From 6182886647a7ad5ce7aaddef59248a9f423895b9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 07:41:06 +0000 Subject: [PATCH 01/11] test: a harness that is the real host Add an in-process harness that runs the plugin inside the real host, so the routing work that follows can be measured against today's behaviour rather than against fakes. Additive only: no src/ change. test/support/host.ts gives two tiers. Tier A is a bare cordis Context with LlmRuntime and llm-pi-ai mounted with { providers }, for the config waterfall and pi-ai's credential paths. Tier B is dsh-app-boot's boot() with a temporary HOME, a test bundle, a fake credentials service and a launch-environment snapshot, for the config editor, settings.describe and the volatile commit. Tier B's rows carry package names through a stub module loader, and it fixes the apply order, because the loader starts rows in parallel. On top of it: - contract tests for the waterfall, the profile and Config shapes, credentials, and the config editor (host-waterfall, host-shapes, host-credentials, host-editor-contracts), plus a tier A smoke test; - a characterization of today's behaviour on pi 1.1.0, each row tagged where P-R1 is expected to flip it, with S0_TODAY shared by the binary e2e; - the keyless 401-vs-404 matrix: a pure classifier, its unit test and a gated live run; - a binary e2e gated on DSH_HOST_E2E=1 with its probe bundle, the e2e-host CI job (a pinned, patched cell on PRs that touch the host seams or vite.config.ts, and a floating cell nightly and on dispatch), and the test:e2e:host script. The binary e2e now reports why a run failed: a timeout only when its own deadline killed the host, the signal for a crash, and the probe's fatal message whenever there is one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- .github/workflows/ci.yml | 152 +++ AGENTS.md | 4 + CONTRIBUTING.md | 56 +- docs/improvement-plan.md | 53 +- package.json | 1 + test/e2e/host-binary.e2e.ts | 478 ++++++++++ test/e2e/host/cordis.patch.yml | 5 + test/e2e/host/package.json | 12 + test/e2e/host/probe-plugin.mjs | 347 +++++++ test/e2e/keyless-matrix.e2e.ts | 185 ++++ test/host-characterization.test.ts | 707 ++++++++++++++ test/host-credentials.test.ts | 144 +++ test/host-editor-contracts.test.ts | 422 +++++++++ test/host-harness.test.ts | 239 +++++ test/host-shapes.test.ts | 461 +++++++++ test/host-waterfall.test.ts | 285 ++++++ test/keyless-matrix.test.ts | 352 +++++++ test/support/host.ts | 1416 ++++++++++++++++++++++++++++ test/support/keyless-matrix.ts | 295 ++++++ test/support/src.ts | 2 + vite.config.ts | 7 +- 21 files changed, 5598 insertions(+), 25 deletions(-) create mode 100644 test/e2e/host-binary.e2e.ts create mode 100644 test/e2e/host/cordis.patch.yml create mode 100644 test/e2e/host/package.json create mode 100644 test/e2e/host/probe-plugin.mjs create mode 100644 test/e2e/keyless-matrix.e2e.ts create mode 100644 test/host-characterization.test.ts create mode 100644 test/host-credentials.test.ts create mode 100644 test/host-editor-contracts.test.ts create mode 100644 test/host-harness.test.ts create mode 100644 test/host-shapes.test.ts create mode 100644 test/host-waterfall.test.ts create mode 100644 test/keyless-matrix.test.ts create mode 100644 test/support/host.ts create mode 100644 test/support/keyless-matrix.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6db0aa08..2f063121 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -234,3 +234,155 @@ jobs: OPENCODE_E2E: "1" OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} OPENCODE_GO_API_KEY: ${{ secrets.OPENCODE_GO_API_KEY }} + + # The published host binary with this checkout linked into a throwaway + # profile: the only run that boots `dsh` itself rather than the host packages + # the unit suite assembles. `test/e2e/host/probe-plugin.mjs` measures the + # registry, the listings, validation, where each turn is sent and the save of + # a catalog-unknown id, and `test/e2e/host-binary.e2e.ts` asserts today's + # rows (`S0_TODAY`, shared with the in-process characterization). + # + # Two cells. `pinned` installs the host with npm `overrides` forcing pi to + # the version `test/support/host-versions.ts` names and applies the + # harness's own patch; `floating` takes whatever pi npm resolves for the + # published host (newer than the harness pins), so a pi release that moves + # the ground shows up nightly. Pull requests run `pinned` only, and only when + # they touch the host seam; dispatch and the nightly run both cells. + # + # The scope is decided by a step rather than by `on.pull_request.paths`: a + # workflow-level path filter would silence every other job in this file. + # Offline: the probe answers every gateway request itself, and the keys it + # stores are dummies. No secret is passed. + e2e-host: + name: e2e · host binary (${{ matrix.cell }}) + if: github.event_name != 'push' + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + cell: ${{ github.event_name == 'pull_request' && fromJSON('["pinned"]') || fromJSON('["pinned","floating"]') }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: does this change touch the host seam? + id: scope + env: + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + run: | + set -euo pipefail + if [ "$EVENT" != "pull_request" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Its own command, so a diff that cannot be computed (a base ref + # that was not fetched, a deleted base branch) fails the step rather + # than reading as "nothing changed" and skipping the job green. + changed="$(git diff --name-only "origin/${BASE_REF}...HEAD")" + # The job's own definition and runner config are in scope: a pull + # request that breaks the install, the override, the patch step or + # the e2e config must run the job it breaks. That includes + # vite.config.ts: it holds the pack config that builds lib/index.mjs + # (the only test that loads that bundle into a real host is this + # one, and its externals keep cordis and schemastery the host's + # copies) and the test settings vitest.e2e.config.ts inherits. + # package.json is not (every dependency bump touches it), so a + # change to its `test:e2e:host` script alone is exercised only by + # the nightly run. + # Until P-R1 moves the host seam under src/host/, the modules that + # install the host patches and mounts whose behaviour depends on the + # real loader and apply order (the hidden-route mount, the listing + # patch and adapter decoration, the Google bridge, the fetch patch, + # the plane bases and the stream hook) are in scope by name. Every + # other row producer (key-capture.ts, models-catalog.ts, config.ts, + # the generated catalog) is covered on each pull request by the + # in-process characterization of the same S0_TODAY rows, and here by + # the nightly run. + if printf '%s\n' "$changed" \ + | grep -Eq '^(src/host/|src/lifecycle\.ts$|src/identity\.ts$|src/responses-provider\.ts$|src/responses-routes\.ts$|src/models-discovery\.ts$|src/google-bridge\.ts$|src/fetch-patch\.ts$|src/pi-provider\.ts$|src/stream-hook\.ts$|cordis\.patch\.yml$|test/e2e/host/|test/e2e/host-binary\.e2e\.ts$|test/support/host\.ts$|test/support/host-versions\.ts$|patches/|\.github/workflows/ci\.yml$|vitest\.e2e\.config\.ts$|vite\.config\.ts$)'; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "no host-seam path changed; the host binary e2e does not run for this pull request" + fi + - if: steps.scope.outputs.run == 'true' + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - if: steps.scope.outputs.run == 'true' + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 26 + cache: pnpm + - if: steps.scope.outputs.run == 'true' + run: pnpm install --frozen-lockfile + - name: the host versions this repository pins + id: versions + if: steps.scope.outputs.run == 'true' + run: | + node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS } from './test/support/host-versions.ts'; console.log('dsh=' + HOST_VERSIONS.dsh); console.log('pi=' + HOST_VERSIONS.pi)" >> "$GITHUB_OUTPUT" + echo "PREFIX=$RUNNER_TEMP/dsh-host" >> "$GITHUB_ENV" + # The pinned install is reproducible, so it is cached by the versions + # and the patch; the floating one is the point of re-resolving, so never. + - name: cache the pinned host install + id: host-cache + if: steps.scope.outputs.run == 'true' && matrix.cell == 'pinned' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/dsh-host + key: dsh-host-${{ matrix.cell }}-${{ steps.versions.outputs.dsh }}-${{ steps.versions.outputs.pi }}-${{ hashFiles('patches/*.patch') }} + - name: install the published host into a throwaway prefix + if: steps.scope.outputs.run == 'true' && steps.host-cache.outputs.cache-hit != 'true' + env: + CELL: ${{ matrix.cell }} + DSH: ${{ steps.versions.outputs.dsh }} + PI: ${{ steps.versions.outputs.pi }} + run: | + mkdir -p "$PREFIX" + # The overrides must be in the prefix's own package.json BEFORE the + # install, or npm resolves the wrapper's ^range to the newest pi. + if [ "$CELL" = pinned ]; then + printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s" } }\n' "$PI" > "$PREFIX/package.json" + else + printf '{ "private": true }\n' > "$PREFIX/package.json" + fi + npm install --prefix "$PREFIX" --no-audit --no-fund "@deepseek-ai/dsh@$DSH" + # Only on a fresh install: re-applying to a cached, patched copy fails. + if [ "$CELL" = pinned ]; then + patch -p1 -d "$PREFIX/node_modules/@earendil-works/pi-ai" < "patches/@earendil-works__pi-ai@$PI.patch" + fi + - name: assert the pi the host resolved + if: steps.scope.outputs.run == 'true' + env: + CELL: ${{ matrix.cell }} + PI: ${{ steps.versions.outputs.pi }} + run: | + resolved="$(node -p "require(process.env.PREFIX + '/node_modules/@earendil-works/pi-ai/package.json').version")" + echo "cell $CELL: pi $resolved" + copies="$(find "$PREFIX/node_modules" -path '*/@earendil-works/pi-ai/package.json' | wc -l)" + if [ "$copies" -ne 1 ]; then + echo "::error::expected one copy of pi under the host prefix, found $copies" + exit 1 + fi + if [ "$CELL" = pinned ] && [ "$resolved" != "$PI" ]; then + echo "::error::the pinned cell resolved pi $resolved, not $PI" + exit 1 + fi + - if: steps.scope.outputs.run == 'true' + run: pnpm run build + - name: boot the host binary with the probe + if: steps.scope.outputs.run == 'true' + run: pnpm run test:e2e:host --run + env: + DSH_HOST_E2E: "1" + DSH_HOST_PREFIX: ${{ runner.temp }}/dsh-host + DSH_HOST_PI_CELL: ${{ matrix.cell }} + DSH_HOST_PROBE_COPY: ${{ runner.temp }}/probe-${{ matrix.cell }}.json + - name: upload the probe's measurement + if: always() && steps.scope.outputs.run == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: host-probe-${{ matrix.cell }} + path: ${{ runner.temp }}/probe-${{ matrix.cell }}.json + if-no-files-found: ignore + retention-days: 7 diff --git a/AGENTS.md b/AGENTS.md index bb28c674..ec36e0f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,6 +43,9 @@ Host bundle (`lib/index.mjs`), a thin `apply` barrel over small modules: - `src/index.ts` · `src/identity.ts` · `src/lifecycle.ts` — the public barrel, the identity constants, and the `apply()` composition (one installer per side effect, in load-bearing order). The barrel is the host's contract and nothing more: `apply`, `Config`, `inject`, `name` and the config types. Everything else is internal, and `test/public-api.test.ts` pins the list, so widening it is a reviewed change. - `test/support/src.ts` — the one door tests use into `src`: each name re-exported from the module that DEFINES it, never through the barrel, so a module move edits one file. `test/architecture.test.ts` enforces both halves — every name is declared in its target, and no test outside the named exemptions (`vi.mock` targets, fresh re-imports after `vi.resetModules()`, the architecture and public-API tests, the jsdom client tests, the `reset-process-state.ts` setup file) imports a `src` path, and no jsdom test loads the door. - `test/support/host-versions.ts` — the host the test install runs (pi, the `@deepseek-ai` cluster, cordis and its loader), named once; `test/host-versions.test.ts` checks the installed tree against it (with `pnpm-lock.yaml` as the record of what pnpm resolved, and the published peer ranges) and `test/harness-channel.test.ts` checks the declarations and the lockfile, including that `@deepseek-ai/schemastery` resolves to one copy inside the cluster's range. +- `test/support/host.ts` — the real host, in-process, in two tiers: **A** (`mountTierA`: a bare cordis `Context`, `LlmRuntime` and `dsh-llm-pi-ai` mounted with `{ providers }`) for the `internal/config` waterfall and pi-ai's credential paths, and **B** (`bootTierB`: dsh-app-boot's `boot()` with a temporary HOME and a test bundle) for the config editor, `settings.describe`, the volatile commit and the characterization of the plugin as it ships. Its fakes (`fakeCredentials`, `fakeLaunchEnvironment`, `recordingFetch`) keep every case offline and its credentials out of the environment (a boot blanks the two keys the plugin reads directly, should the shell export them, and restores them on dispose); `S0_PROFILE` is the contract fixture, `CHARACTERIZED_PROFILE` (S0 plus a declared catalog-unknown id) the characterization's, and `S0_TODAY` the measured per-model rows, tagged per facet, that the binary e2e shares (`todayRowsFor` picks them by the apply order the probe infers from whether the plugin's decoration wrapped pi-ai's plane adapter). Tier A has no loader, so this plugin mounted there cannot find pi-ai and mounts no hidden route and no Google bridge (whose adapter class comes from the same loader entry, `loadPiAiAdapterHost`); `test/host-harness.test.ts` exercises the builder options no contract or characterization case passes, and tier B's two failure paths (an invalid first row, an exported key). Tier B's rows carry PACKAGE names through a stub module loader, never `cordis:` builtins, because the plugin finds pi-ai's entry by `options.name` (`piAiEntry` in `src/responses-provider.ts`) and a builtin-named row would drop the hidden-route mount from the characterization. It also fixes which of pi-ai and the plugin applies first (`order`), because the loader starts rows in parallel and the published binary applies the plugin first; the held row is released when the first one applies or its fiber settles, so an invalid first row fails instead of hanging the boot. +- `test/support/keyless-matrix.ts` — the keyless 401-vs-404 probe matrix's cell table and classifier. Pure, though it lives in `test/support/` rather than `src/utils/`: a function of its arguments, with no code path that can set a credential header; `test/e2e/keyless-matrix.e2e.ts` sends the cells. +- `test/e2e/host/` — the measurement bundle (`probe-plugin.mjs`, its manifest and patch) the `e2e-host` job links into a throwaway profile under a temporary HOME, through a relative `link:` path. `test/e2e/host-binary.e2e.ts` is its reader, gated on `DSH_HOST_E2E=1`; it installs no host itself and needs `DSH_HOST_PREFIX`, a prefix holding the installed one. - `patches/` — the harness's own pi patch, copied verbatim and applied by `patchedDependencies` in `pnpm-workspace.yaml`. Its hash is pinned by `test/host-versions.test.ts` and recorded in `pnpm-lock.yaml`, so never hand-edit it: re-copy it from the harness and re-run `pnpm install`. - `src/config.ts` · `src/config-values.ts` · `src/utils/guards.ts` — the schema and its defaults; dependency-free readers and type guards. The client bundle may import these, and it does — along with the contract and presentation modules (`usage-contract.ts`, `login-contract.ts`, `settings-copy.ts`, `settings-fields.ts`, `usage-ui.ts`, `go-limits.ts`). The rule is not a list of names: **the client may import a module that reaches nothing host-only** — no cordis context, no credential seam, no node built-in — and the bundle gate is what enforces it, by failing the build rather than by anyone remembering. An earlier version of this line named two files as the only ones allowed, which the tree had already outgrown. @@ -108,6 +111,7 @@ Web client bundle (`lib/client.js`): `src/settings-page.tsx` (entry, wiring only vp test run # unit suite (refuses to start below engines.node) pnpm run test:shuffle # the unit suite in random order; CI runs it on Node 24 pnpm run test:mock # gateway mock suite: offline, no key, runs in CI +pnpm run test:e2e:host # opt-in: the published host binary with the probe (CONTRIBUTING, "The e2e-host job") vp test run --coverage # with the coverage ratchet (95/90/93/95) vp check # format, lint, types vp fmt --write # rewrite formatting diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9babbc54..66529591 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,7 @@ pnpm run test:shuffle # the same run in a random order; reproduce a failure w pnpm run test:mock # gateway mock suite: offline, loopback only, runs in CI pnpm run build # vp pack + client rename -> lib/index.mjs, lib/index.d.mts, lib/client.js pnpm run test:e2e # opt-in end-to-end suite; talks to the real OpenCode gateway +pnpm run test:e2e:host # opt-in: the published host binary with the probe (see "The e2e-host job") pnpm run catalog:shim # regenerate src/catalog-data.ts from models.dev ``` @@ -34,13 +35,62 @@ OPENCODE_E2E=1 OPENCODE_API_KEY=… OPENCODE_GO_API_KEY=… pnpm run test:e2e - `OPENCODE_E2E=1` is required; without it every case skips, so a bare `pnpm run test:e2e` is a safe no-op. - `OPENCODE_API_KEY` (Zen) arms the live `/models` enrichment case and the **paid** protocol-routing cases; `OPENCODE_GO_API_KEY` (Go plan) arms the live `/usage` case. Each block skips when its key is absent, which is why the CI `e2e` job stays green on fork PRs (secrets are not exposed to them) while still checking the endpoints answer. -- The **free-tier and unmetered** routing cases run keyless on purpose: measured 2026-10-06, a paid model answers `401 AuthError` on every endpoint without a credential, so keyless probing cannot tell them apart at all. Keyless discrimination is real only for the free classes. -- `OPENCODE_ZEN_BASE_URL` / `OPENCODE_GO_BASE_URL` retarget the suite at a mirror. +- The **free-tier and unmetered** routing cases run keyless on purpose: measured 2026-10-06, a paid model answers `401 AuthError` on every endpoint without a credential, so keyless probing cannot tell the right PROTOCOL endpoint from a wrong one. Keyless protocol discrimination is real only for the free classes. What a paid model's keyless answer does still tell is whether a PATH exists at all — a removed path answers 404 — which is the one thing the keyless matrix below reads. +- `OPENCODE_ZEN_BASE_URL` / `OPENCODE_GO_BASE_URL` retarget `opencode-live` and `protocol-routing` at a mirror. The keyless matrix measures the vendor's own paths, which a mirror's base cannot name, so it skips while either is set. -Two files, two jobs: +What the main files are for: - `opencode-live.e2e.ts` — the only place the plugin meets the real API. It catches what a stub cannot: **the vendor changing the payload**. The meter parses `/zen/go/v1/usage` and the picker consumes an enriched `/models` listing, so both shapes are asserted against the live service. The unkeyed cases assert reachability (never a 404) and that our header set does not change the gateway's verdict on a request — compared against the same call without it, rather than hardcoding a status the vendor may tighten. - `patched-fetch-headers.e2e.ts` — a local `node:http` listener, so the _outgoing_ header set crosses a real socket. The live gateway cannot echo a request back; everywhere else in the suite the injected headers are asserted against a captured `fetch`. +- `keyless-matrix.e2e.ts` — the keyless 401-vs-404 matrix: two planes × four protocols × the legacy (`/zen(/go)/v1`) and modern (`/inference(/go)/…`) endpoint families, one PAID model per cell from the live catalog — never a free one, which the gateway serves keyless with a real completion. It **sends no key and reads none**, so the `e2e` job runs it on every pull request, and it retries a cell once on a 429, a 5xx or a transport error before reading it. A 404 means the path is gone, a 401/403 that it exists, a 410 or a 400 "Model is unavailable" that the model is. It asserts that all sixteen cells have a model and were sent, that every answer is readable, and that no cell the plugin dispatches to today with the default endpoint (`auto`, an alias of `modern`; `TODAY_DISPATCH`: completions on the LEGACY family, Responses, Messages and Google on the modern one) is gone — nor the modern completions cells P-R1 moves completions to (`NEXT_DISPATCH`, a labelled forward guard). The other legacy verdicts are recorded for P-R2 — among them the cells an `endpoint: legacy` user dispatches to (Responses and Messages on both planes, and Zen's Google), which are recorded, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell `TODAY_DISPATCH` already asserts. The cell table, both sets and the classifier are a pure module (`test/support/keyless-matrix.ts`), unit-tested offline in `test/keyless-matrix.test.ts`, which also derives `TODAY_DISPATCH` from the URLs `S0_TODAY` measured. +- `protocol-routing.e2e.ts` — asks the gateway which endpoint serves each model's wire format: keyless for the free classes, keyed (and skipped without a secret) for paid ones. +- `dispatch-levels.e2e.ts` — every thinking level the picker offers is one pi's own rule accepts, swept over the whole shipped catalog; no network. +- `refusal-classification.e2e.ts` — the refusal classifier's verdicts on the gateway's bytes over a real `node:http` socket. +- `host-binary.e2e.ts` — the published host binary; see [The real-host harness](#the-real-host-harness). It skips unless `DSH_HOST_E2E=1`. + +## The real-host harness + +`test/support/host.ts` builds the host from the packages this repository installs, at the versions `test/support/host-versions.ts` names (pi 1.1.0 with the harness patch, the `@deepseek-ai` cluster at the published host's tag). The routing work is judged against it: the contract tests pin the host facts the design rests on (`docs/improvement-plan.md`, F11–F20), and the characterization rows record what today's plugin does. + +Two tiers, both in-process: + +- **Tier A — `mountTierA`.** A bare cordis `Context`, `LlmRuntime`, and `dsh-llm-pi-ai` mounted as a plugin with `{ providers }` (and optionally this plugin). No loader and no files. Use it for the `internal/config` waterfall and pi-ai's credential paths — anything that needs a fiber and the registry and nothing that reads a profile (`host-waterfall`, `host-shapes`, `host-credentials`). Mounted here, this plugin is a **degraded** one: it finds pi-ai through the loader's entries, so with no loader it patches fetch but mounts no hidden route and no Google bridge — the bridge's adapter class comes from the loader's pi-ai entry too (`host-harness` pins both: the stand-down line, and `loadPiAiAdapterHost` answering `undefined`). Characterize the real plugin in tier B. +- **Tier B — `bootTierB`.** dsh-app-boot's `boot()` with a temporary HOME, a test bundle and a profile patch, the way the harness's own composition tests boot one. Use it for the config editor, `settings.describe`, the loader's volatile commit and the characterization of the plugin as it ships (`host-editor-contracts`, `host-characterization`). Its rows carry **package names**, resolved by a stub module loader, not `cordis:` builtins: the plugin finds pi-ai's entry by `options.name === "@deepseek-ai/dsh-llm-pi-ai"` (`src/responses-provider.ts`), so a builtin-named row would silently drop today's hidden-route mount and the Google bridge. The loader starts rows in parallel, so tier B makes the order explicit: `order: "plugin-first"` (the default, and what the published binary does) or `"pi-ai-first"`. + +What every case does: + +- **Install the recording fetch before mounting** (`vi.stubGlobal("fetch", recordingFetch().fetch)`) and restore it with `vi.unstubAllGlobals()` after disposal: the plugin's fetch patch wraps whatever `fetch` is current when it mounts. The recording fetch answers `opencode.ai` and `models.opencode.ai` with a 401, refuses every other host, and keeps credential headers by length only. +- **Dispose in `afterEach`** (or a `finally`), and boot per case: a key the fetch patch captured in one case would change the next one's listing. +- **Credentials come from the fakes, not the environment.** The fakes (`fakeCredentials`, `fakeLaunchEnvironment`) are provided on the root, which is where pi-ai looks first. The plugin reads `OPENCODE_API_KEY`/`OPENCODE_GO_API_KEY` from the environment directly, so `bootTierB` blanks either one your shell exports for the life of the boot and restores it on dispose: the suite measures the keyless deployment whatever the shell holds (`host-harness` pins it). A case never sets them itself. +- **Give tier-B cases 30 s**, and wait with a deadline (`waitForRoutes`, 10 s by default, so a route that never registers fails by name inside the hook's budget), never a fixed sleep: the plugin mounts its hidden routes asynchronously. +- **A first row that fails does not hang the boot.** Tier B holds the second of pi-ai and the plugin until the first applies OR its fiber settles in any state, so a row whose config the schema rejects boots with that fiber FAILED (`fiber.await()` rethrows the real error) and the other row applied. + +**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it) or `[INTENDED_CHANGE]` (right today; a phase changes it on purpose). A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY` in `host.ts`, which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, so the tags in `S0_TODAY` are what govern which entries a phase may flip. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. + +**A catalog refresh can touch the rows.** `S0_TODAY`'s context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case compares each row's context window and effort ids to the bundled catalog's derivation and names the model that moved; when it fails on a refresh pull request, update the row in `S0_TODAY`. A phase that moves a non-`BASELINE` `info` facet off the catalog on purpose (the user's context window, a toggle model's Off/On) adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. + +**The apply order is a race on the real host.** Tier B fixes it; the published binary does not. The rows depend on whether the plugin's adapter decoration wrapped pi-ai's plane adapter, so the probe records that and the order it implies (`inferredOrder` — inferred, not observed: a pi-ai restart after the plugin applied would read as plugin-first), and the binary e2e picks its rows with `todayRowsFor(inferredOrder)` instead of pinning one. + +### The `e2e-host` job + +`.github/workflows/ci.yml`'s `e2e-host` job installs the published `@deepseek-ai/dsh` (at `HOST_VERSIONS.dsh`) into a throwaway prefix, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the profile's credentials file, boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. + +- **Cells.** `pinned` forces pi to `HOST_VERSIONS.pi` with npm `overrides` and applies the harness patch with `patch -p1`, then asserts every row. `floating` takes the pi npm resolves for the published host. It skips the rows that depend on which ids pi describes (the keyed listing and the per-model rows) and still asserts the save rows. Those assume the floating pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage a red floating run as a pi change first. Its stored-gap save row asserts only that SOME undescribed id blocks the save, not which one; its probe JSON is the record. +- **Triggers.** Dispatch and the nightly run take both cells. A pull request takes `pinned`, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` or `patches/` (what the job installs and patches), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. + +To run it locally, install the host into a prefix outside the checkout, then point the suite at it (placeholder paths; `pnpm` must be on `PATH`). The first line reads the versions from `test/support/host-versions.ts`, the same way the CI job does, so the snippet never names one itself: + +```sh +eval "$(node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS as v } from './test/support/host-versions.ts'; console.log('DSH=' + v.dsh + ' PI=' + v.pi)")" +mkdir -p /home/you/dsh-host +printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s" } }\n' "$PI" > /home/you/dsh-host/package.json +npm install --prefix /home/you/dsh-host "@deepseek-ai/dsh@$DSH" +patch -p1 -d /home/you/dsh-host/node_modules/@earendil-works/pi-ai < "patches/@earendil-works__pi-ai@$PI.patch" +pnpm run build +DSH_HOST_E2E=1 DSH_HOST_PREFIX=/home/you/dsh-host pnpm run test:e2e:host --run +``` + +`DSH_HOST_PI_CELL=floating` runs the floating cell against an install without the override and the patch. ## Live profile loop diff --git a/docs/improvement-plan.md b/docs/improvement-plan.md index 74f6937f..1ca3f82f 100644 --- a/docs/improvement-plan.md +++ b/docs/improvement-plan.md @@ -23,7 +23,7 @@ Phases land as **stacked pull requests**. Each one is based on the previous phas | P1b | `chore:` hygiene | in review | | P2 | `feat:` a smaller public API | in review | | P-T1a | `build:` pin the host | in review | -| P-T1b | `test:` a harness that is the real host | planned | +| P-T1b | `test:` a harness that is the real host | in review | | P-H | `refactor:` one way to patch a host method | planned | | P-R1 | `feat:` the plugin owns the plane routes | planned | | P-R2 | `fix:` the endpoint family, measured | planned | @@ -321,15 +321,16 @@ What stays private is pinned by contract tests against the real host packages: - **Pinning phases open with a green pin**, which the PR shows unchanged: P-T1a/b, P-H, P5, P7, P8, P9, P10a and P-T2. - **P-T3 and P12 are exempt.** -**Characterization rows are tagged when they are recorded**, so a later phase flips named rows: +**Characterization rows are tagged when they are recorded**, so a later phase flips named rows. The authoritative per-facet table is `S0_TODAY` in `test/support/host.ts` (each model's info, call-config verdict and wire carry their own tag); a phase's PR body cites the rows it flips from there. The three tags: -- `KNOWN_DEFECT`: - - Go Gemini answers `INVALID_CONFIG`; +- `BASELINE`: right today and must survive every phase. No phase may move a `BASELINE` facet, except that P-R2's measured family flip, if it happens, may move the endpoint-family part of a wire facet; it re-tags each one it moves and names it in its PR body. +- `KNOWN_DEFECT`, wrong today and a phase fixes it, for example: + - Go Gemini answers `INVALID_CONFIG` — in the published host's apply order (`plugin-first`); with pi-ai applying first it validates, so the row is order-dependent (`todayRowsFor`); - an unknown id is rejected; - toggle models show Minimal/Xhigh/Max. -- `INTENDED_CHANGE`: +- `INTENDED_CHANGE`, right today and a phase changes it on purpose, for example: - completions models move from `/zen(/go)/v1` to `/inference(/go)/openai/v1`; - - `decoratePlaneAdapters`' baseUrl repair never reaches the wire. + - `decoratePlaneAdapters`' baseUrl repair never reaches the wire — a row of the `pi-ai-first` block only: in `plugin-first` the decoration never runs, and that row is a `KNOWN_DEFECT`. 1. **P1a `fix:` shipped breakage.** - The wrapper's broken `export { default }`. @@ -357,15 +358,24 @@ What stays private is pinned by contract tests against the real host packages: - The `@deepseek-ai/*` packages move to rc.2, and the devDependencies gain `dsh-llm`, `dsh-app-boot`, `dsh-config-editor`, `dsh-settings` and `cordis-plugin-loader`. - Add `test/support/host-versions.ts`, with a version test: `loadHostPi().version` is 1.1.0, and the root pi is the same copy. - Rewrite `mount-protocols.test.ts` as "every catalog SDK maps to a protocol that pi's `api/*` exports implement", keeping the row that pi 1.1.0's Go provider lacks Google. -5. **P-T1b `test:` the real-host harness.** It is additive only. `test/support/host.ts` provides two tiers that run inside the test process: - - **Tier A:** a cordis `Context`, `LlmRuntime`, and pi-ai mounted with `{ providers }`. - - **Tier B:** dsh-app-boot's `boot()` with a temporary HOME, and a test bundle that names the loader builtins `llm`, `llm-pi-ai`, `ConfigEditor`, `Settings` and the plugin. It follows the harness's `experimental/speech-to-text/tests/configuration.spec.ts:7-33`, with a fake `credentials` service and a launch-environment snapshot. +5. **P-T1b `test:` the real-host harness.** Additive only; in review. It landed `test/support/host.ts` (tier A: `Context` + `LlmRuntime` + pi-ai; tier B: app-boot `boot()` with a temporary HOME), the contract tests (`host-waterfall`, `host-shapes`, `host-credentials`, `host-editor-contracts`), the tagged characterization (`host-characterization`, with `S0_TODAY` shared by the binary e2e), the keyless matrix, the probe bundle and the `e2e-host` job. Two departures from the phase as first planned, both deliberate: tier B's rows carry package names through a stub module loader rather than `cordis:` builtins, because the plugin finds pi-ai by `options.name` (`piAiEntry` in `src/responses-provider.ts`); and tier B fixes the apply order (`plugin-first`, the binary's, or `pi-ai-first`), because the loader starts rows in parallel. Two narrower ones: the `e2e-host` PR scope is wider than first named (the job's own inputs, and until P-R1 the `src` modules that install the host patches and mounts whose behaviour depends on the real loader and apply order), and the floating cell skips the rows that depend on which ids pi describes. - This phase lands: - - the contract tests; - - the tagged characterization rows; - - the S0 assertions, including `settings.describe`, `hasProviderApiKey` and the Models store's join; - - the CI additions below. + The `hasProviderApiKey` baseline (`api/session-controller/src/catalog.ts:97-116`) is a **restatement**: session-controller does not export the function (its `exports` map reaches `SessionController` and `buildModelCatalog` only), so `test/support/host.ts` rebuilds it line for line (`providerWithApiKey`) from the real parts tier B holds — `settings.describe({ redactSecrets: true })`, `llm.listConfigurableProviders()` and `credentials.describe` — and both `host-editor-contracts` and `host-characterization` use it. The cases in `host-editor-contracts` boot WITHOUT the plugin, so they are host contracts that nothing in `src/` can move: with only the plane keys stored it answers true (the Zen plane); under a test-local strip listener it answers false, because the planes' `describe().value` is gone (F20), which upstream makes `initializeDefaultModel` replace the user's default model. That false is the host fact the D18 wrapper exists to counter, not a row any phase flips. The guard on the plugin is the `[BASELINE]` row in `host-characterization`: today's plugin as it ships, with only the plane keys stored, in both apply orders, answers the Zen plane, and pi-ai's `settings.describe` value lists both planes. P-R1 must keep that row green while its own strip is active. + + **What remains:** the Models store's join. It lives in the client package `ui-settings-models`, which is not installed here and has no server-side seam to restate, so P-R1 owns it (its contract list below, unconditionally), measured on the binary or with the package added at the pinned cluster tag. + + The tier-A F11 smoke (`listProviders()` equals pi-ai's resolved keys) lives in `host-waterfall`, beside the other F-numbered contracts; `host-harness` holds the builder-seam cases. + + **Corrections the harness measured** (P-R1 must design for these): + - **F8/F12, the save check.** An outermost listener that strips the planes and returns a copy does NOT shield pi-ai's own listener: pi-ai validates the value `next()` returns, which is the raw value the inner chain saw (`llm-pi-ai/src/index.ts:177-186`). Against its stripped previous config every plane entry reads as changed, so the strict check runs on the planes: an editor save is refused while a plane stores any gap id. The claim listener must either keep pi-ai's listener from seeing the planes (run the nested waterfall on the copy, with a re-entrancy guard) or veto the chain for pi-ai's fiber and validate in its place. + - **F16, `!!js` in a provider.** pi-ai's listener runs `Config(raw)` on that same un-interpolated value, so a `!!js` node inside `providers` boots (evaluated) but makes every later save of the row fail with `expected string but got [object Object]`. The F16 pin is therefore taken on the plugin's own row. + - **`update()` runs the waterfall twice**, both as commits (`raw === _config`): once while ACTIVE with pi-ai's listener present (`fiber.ts:747`), once on the restart without it (`fiber.ts:655`). A first load and a `restart()` run it once, without pi-ai's listener. + - **F15 holds** for a plane-only edit under the strip: no `loader/volatile-update`, no `llm/adapters-updated`, no restart. A sibling's edit emits one volatile update and two registry events (route replace and directory replace, `llm/src/index.ts:475,536`). + - **The apply order changes today's rows.** On the published binary the plugin applies about 70 ms before pi-ai registers, so `decoratePlaneAdapters` (which wraps only what is registered when its effect runs) never wraps pi-ai's plane adapter. In that order Go `gemini-4-argon-contributor` answers `INVALID_CONFIG` — the plan's `KNOWN_DEFECT` holds — while the turn is still served on the Go Google base through the bridge. With pi-ai first it validates, and the Go plane's repaired base is Zen's (both planes share one adapter instance, wrapped twice). + - **Zen `gpt-5.3-codex-spark`** is refused by `resolveCallConfig` (`INVALID_CONFIG`) yet dispatched to `/inference/openai/v1/responses` through the hidden route, and the user's `name`/`contextWindow` never reach `resolveModelInfo` (the catalog's 128000 does). + - **Listings.** Before a key is captured the listing patch hides every paid model on both planes (`[]`); after one, only pi-resolved ids are listed and the gap ids are not merged back. + - **A stored gap id blocks every edit of its plane**, `displayName` included: the strict check names the stored id first, ahead of the one being saved. + - **Shapes.** A resolved profile also carries the profile's pass-through fields (`compat`, `defaultContextWindow`, `defaultInput`, `defaultMaxTokens`, `headers`, `modelOverrides`, `thinkingBudgets`), and a built pi provider exposes `getModels()` but no `api` map, so F10 is pinned by behaviour (pi's own "no API implementation" error) rather than by its keys. 6. **P-H `refactor:` one way to patch a host method.** `host/patch-method.ts` and `test/reload.test.ts` (apply A → apply B → dispose A → B intact → dispose B → originals). 7. **P-R1 `feat:` the plugin owns the plane routes.** @@ -385,7 +395,7 @@ What stays private is pinned by contract tests against the real host packages: - **The PR body** names every tagged row it flips. 8. **P-R2 `fix:` the endpoint family, measured.** - Probe 3 URL shapes × 4 protocols × 2 planes, keyless, plus keyed probes per family with the CI keys. - - Set the default family and pin it with a test. If a `modern` cell fails, flip the default; `endpoint: legacy` still maps every protocol. + - Set the default family and pin it with a test. If a `modern` cell fails, flip the default; `endpoint: legacy` still maps every protocol except Go's Google, which stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`). Decide here whether `endpoint: legacy` is a guarded configuration: the keyless matrix records its Responses and Messages cells and Zen's Google cell but does not assert them. 9. **P3 `feat:` Settings complements pi's list.** - The `discoverModels` wrapper answers servable, non-deprecated models, deduped and whitelisted. - Delete `installModelDiscovery` and `toDiscovered`. @@ -468,8 +478,9 @@ P0 → P1a → P1b → P2 → P-T1a → P-T1b → P-H → P-R1 → P-R2 → P3 - **Real host, tier B:** - `configEditor.edit` writes the raw `next` to the YAML; - `_commitVolatile` re-resolves through a prepended listener; - - baselines for `settings.describe` and `hasProviderApiKey`. -- **Characterization rows**, tagged, on pi 1.1.0. + - the `settings.describe` and `hasProviderApiKey` host contracts, the latter restated from the real parts (above), with the with-plugin `[BASELINE]` row in the characterization; the Models store's join moves to P-R1. +- **Characterization rows**, tagged, on pi 1.1.0, in both apply orders. +- **Binary e2e and the keyless matrix**, as below. **P-R1** @@ -550,7 +561,7 @@ P0 → P1a → P1b → P2 → P-T1a → P-T1b → P-H → P-R1 → P-R2 → P3 ### The `e2e-host` job (`ci.yml`) -**When it runs:** on dispatch, nightly, and on PRs that touch `src/host/**`, `src/lifecycle.ts`, `src/identity.ts` or `cordis.patch.yml`. +**When it runs:** on dispatch, nightly, and on PRs that touch `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each PR by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts`, `test/support/host-versions.ts`, `patches/**`, `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the pack config that builds `lib/index.mjs`, which only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step, since a workflow-level path filter would silence every job). **What it does:** @@ -561,12 +572,12 @@ P0 → P1a → P1b → P2 → P-T1a → P-T1b → P-H → P-R1 → P-R2 → P3 **Cells:** -- (a) pi 1.1.0, pinned and patched, on PRs; -- (b) whatever npm installs, nightly and on dispatch. +- (a) pi 1.1.0, pinned and patched, on PRs, nightly and on dispatch; it asserts every row; +- (b) whatever npm installs, nightly and on dispatch. It skips the rows that depend on which ids pi describes (the keyed listing and the per-model rows) and still asserts the save rows. Those assume the floating pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design. Its probe JSON is the record. ### The live gateway job (`OPENCODE_API_KEY` / `OPENCODE_GO_API_KEY`) -- **Every PR:** the keyless matrix. A 404 means the path is gone; a 400 "Model is unavailable" or a 410 means the model is gone. +- **Every PR:** the keyless matrix. A 404 means the path is gone; a 400 "Model is unavailable" or a 410 means the model is gone. It asserts that every cell has a model and was sent, and that no cell today's plugin dispatches to with the default endpoint (`auto`, an alias of `modern`: legacy completions; modern Responses, Messages and Google) nor P-R1's modern completions target answers 404; the other legacy verdicts are P-R2's record. `endpoint: legacy` users dispatch Responses and Messages on both planes, and Zen's Google, to the legacy cells, which are recorded for P-R2, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell already asserted. - **PRs that touch `src/host/**`, and nightly:** minimal keyed probes, one per plane × protocol, with a small `max_tokens`. - **Nightly and on dispatch:** the full keyed suites. diff --git a/package.json b/package.json index d807ce0c..92dfb9a5 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "test": "vp test", "test:coverage": "vp test run --coverage", "test:e2e": "vp test --config vitest.e2e.config.ts", + "test:e2e:host": "vp test --config vitest.e2e.config.ts test/e2e/host-binary.e2e.ts", "test:mock": "vp test --config vitest.mock.config.ts", "test:shuffle": "vp test run --sequence.shuffle", "typecheck": "tsc --noEmit", diff --git a/test/e2e/host-binary.e2e.ts b/test/e2e/host-binary.e2e.ts new file mode 100644 index 00000000..76fcf5e2 --- /dev/null +++ b/test/e2e/host-binary.e2e.ts @@ -0,0 +1,478 @@ +/** + * The published host binary, booted with this plugin and the measurement probe. + * + * The in-process tiers (`test/support/host.ts`) assemble the host from the + * packages this repository installs; this file boots the real thing — the + * `@deepseek-ai/dsh` the `e2e-host` job installs into a throwaway prefix — the + * way a user's profile does: a temporary HOME, a profile whose `package.json` + * links this checkout and the probe bundle (`test/e2e/host/`) through RELATIVE + * `link:` paths, `dsh plugin … install`, then `dsh --profile e2e --no-open`. + * The probe records what the host serves and exits the process; this file reads + * its JSON. + * + * The assertions are TODAY's behaviour, read from the same `S0_TODAY` table the + * in-process characterization pins, so the binary and the harness cannot drift + * apart unnoticed. P-R1 flips the rows its PR names. The loader starts rows in + * parallel, so which of pi-ai and the plugin applies first is a race. The rows + * depend on whether the plugin's adapter decoration wrapped pi-ai's plane + * adapter, so the probe records that and the order it implies + * (`inferredOrder`), and the per-model rows follow it (`todayRowsFor`) rather + * than pinning a 70 ms margin. + * + * Gated: it runs only with `DSH_HOST_E2E=1`, and needs `DSH_HOST_PREFIX`, a + * directory whose `node_modules/@deepseek-ai/dsh` is the installed host. + * `DSH_HOST_PI_CELL=pinned` (the default) also requires the pi + * `host-versions.ts` names, with the harness patch, and asserts every row; + * `floating` accepts whatever pi the host resolved. It skips the rows that + * depend on which ids pi describes (the keyed listing and the per-model rows) + * and still asserts the save rows. Those assume the floating pi lacks at least + * one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so + * a pi that learns all of them or retires a declared id turns them red by + * design — triage a red floating run as a pi change first. + * + * `DSH_HOST_PROBE_COPY` names a file to keep the probe's JSON in, for the CI + * artifact; it is written whenever the probe wrote one, a failed run included. + * `pnpm` must be on `PATH`, and `pnpm run build` must have run. + * The keys the profile names are DUMMY values written to the temporary + * credentials store, so every turn reaches the probe's 401 and nothing real is + * ever sent. + * + * @module test/e2e/host-binary.e2e + */ +import { spawn, spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative, sep } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { HOST_VERSIONS } from "../support/host-versions.ts"; +import { + type ApplyOrder, + CHARACTERIZED_MODELS, + CHARACTERIZED_PROFILE, + PI_AI_PACKAGE, + S0_TODAY, + SAVE_BLOCKED_BY_ANY_UNDESCRIBED, + SAVE_CANDIDATE, + todayRowsFor, + ZEN_UNDESCRIBED, +} from "../support/host.ts"; +import { INTERNAL_ROUTES } from "../support/src.ts"; + +const ENABLED = process.env.DSH_HOST_E2E === "1"; +const PREFIX = process.env.DSH_HOST_PREFIX ?? ""; +const CELL = process.env.DSH_HOST_PI_CELL ?? "pinned"; +const PINNED = CELL === "pinned"; +const BOOT_DEADLINE_MS = 180_000; +const CHECKOUT = fileURLToPath(new URL("../..", import.meta.url)); +const PROBE_BUNDLE = join(CHECKOUT, "test", "e2e", "host"); + +/** What the probe writes (test/e2e/host/probe-plugin.mjs). */ +interface ProbeOutput { + readonly fatal?: string; + readonly fetchInnermost: boolean; + readonly routesSettled: boolean; + readonly planeAdapterDecorated: boolean; + /** Inferred from `planeAdapterDecorated`, not observed: see the probe. */ + readonly inferredOrder: ApplyOrder; + readonly providers: string[]; + readonly registryRoutes: string[]; + readonly owners: Record; + readonly configurable: { provider: string; settingsNs: string }[]; + readonly listModels: Record; + readonly listModelsAfter: Record; + readonly modelInfo: Record< + string, + { context: number; efforts: string[] } | { error: string } + >; + readonly callConfig: Record; + readonly dispatch: Record; + readonly describe: { user: string[]; value: string[] } | undefined; + readonly saveBlocked: { id: string; rejected?: string; accepted?: true }; + readonly saveCleared?: { + dropped: string[]; + rejected?: string; + accepted?: true; + }; + readonly save: { id: string; rejected?: string; accepted?: true }; + readonly afterSave?: unknown; +} + +/** The entries of a directory, or none when it cannot be read. */ +const entriesOf = (dir: string): string[] => { + try { + return readdirSync(dir).filter((entry) => !entry.startsWith(".")); + } catch { + return []; + } +}; + +/** + * Every `package.json` of pi under a `node_modules` tree, at any depth: the + * same set as CI's `find -path '*\/@earendil-works/pi-ai/package.json'`. + * Every package's own `node_modules` is walked, scoped ones included — pi-ai's + * and its `@earendil-works` siblings' too, where a nested copy would hide. + */ +const piCopies = (root: string): string[] => { + const found: string[] = []; + const walkPackage = (dir: string, scope: string, name: string): void => { + const manifest = join(dir, "package.json"); + if ( + scope === "@earendil-works" && + name === "pi-ai" && + existsSync(manifest) + ) { + found.push(manifest); + } + walk(join(dir, "node_modules")); + }; + const walk = (dir: string): void => { + for (const entry of entriesOf(dir)) { + const full = join(dir, entry); + if (entry.startsWith("@")) { + for (const scoped of entriesOf(full)) { + walkPackage(join(full, scoped), entry, scoped); + } + } else { + walkPackage(full, "", entry); + } + } + }; + walk(root); + return found; +}; + +/** + * How the host process ended. `timedOut` is set only by this file's own + * deadline timer, so a signal the test did not send (a V8 abort, the runner's + * OOM killer) never reads as a timeout. + */ +interface RunEnd { + code: number | null; + signal: NodeJS.Signals | null; + timedOut: boolean; +} + +/** + * Why the run failed, from how it ended and what it left behind, for the + * failure message: the real cause first, so a timeout never reads as an exit + * code, a crash never reads as a timeout, and a complete result never reads + * as an unreadable one. + * + * @param end - how the process ended. + * @param written - the probe file's text, if it was written. + * @param parsed - that text parsed, if it parsed. + * @returns one sentence naming the cause. + */ +const whyRunFailed = ( + end: RunEnd, + written: string | undefined, + parsed: ProbeOutput | undefined +): string => { + let how = `exited ${String(end.code)}`; + if (end.timedOut) { + how = `timed out after ${BOOT_DEADLINE_MS} ms and was killed`; + } else if (end.signal !== null) { + how = `was killed by ${end.signal}`; + } + if (parsed?.fatal !== undefined) { + return `the host ${how}, with the probe's fatal error: ${parsed.fatal}`; + } + let left = "after writing a complete probe result (see the artifact)"; + if (written === undefined) { + left = "without a probe result"; + } else if (parsed === undefined) { + left = "with an unreadable probe result"; + } + return `the host ${how}, ${left}`; +}; + +/** POSIX separators, so a `link:` spec reads the same on every runner. */ +const posix = (path: string): string => path.split(sep).join("/"); + +let home = ""; +let out: ProbeOutput; + +describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { + beforeAll(async () => { + if ( + PREFIX.length === 0 || + !existsSync(join(PREFIX, "node_modules", "@deepseek-ai", "dsh")) + ) { + throw new Error( + "DSH_HOST_PREFIX must name a directory whose node_modules/@deepseek-ai/dsh is the installed host" + ); + } + const bin = join( + PREFIX, + "node_modules", + "@deepseek-ai", + "dsh", + "lib", + "bin.js" + ); + const piRoot = join(PREFIX, "node_modules", "@earendil-works", "pi-ai"); + const piVersion = ( + JSON.parse(readFileSync(join(piRoot, "package.json"), "utf8")) as { + version: string; + } + ).version; + console.info(`[e2e-host] cell ${CELL}: pi ${piVersion}`); + expect(piCopies(join(PREFIX, "node_modules"))).toHaveLength(1); + if (PINNED) { + expect(piVersion).toBe(HOST_VERSIONS.pi); + // The harness patch is applied (the check test/host-versions.test.ts runs). + expect( + readFileSync(join(piRoot, "dist", "api", "pi-messages.js"), "utf8") + ).not.toContain("parseStreamingJson(json)"); + } + expect(existsSync(join(CHECKOUT, "lib", "index.mjs"))).toBe(true); + + home = mkdtempSync(join(tmpdir(), "dsh-host-e2e-")); + const dshHome = join(home, ".dsh"); + const profileDir = join(dshHome, "profiles", "e2e"); + // No inherited npm_config_* and no OPENCODE_* key: the run sees only what + // this file gives it. + const env = { + CI: "1", + DSH_HOME: dshHome, + DSH_TELEMETRY_DISABLED: "1", + HOME: home, + PATH: process.env.PATH ?? "", + }; + const dsh = (args: string[]) => { + const run = spawnSync(process.execPath, [bin, ...args], { + cwd: home, + encoding: "utf8", + env, + timeout: BOOT_DEADLINE_MS, + }); + if (run.status !== 0) { + // A spawn error, a timeout and a signal all leave `status` null; name + // the one that happened rather than "exited null". + let how = `exited ${String(run.status)}`; + if (run.error !== undefined) { + how = `failed: ${run.error.message}`; + } else if (run.signal !== null) { + how = `was killed by ${run.signal}`; + } + throw new Error( + `dsh ${args.join(" ")} ${how}:\n${(run.stderr ?? "").slice(-2000)}` + ); + } + }; + // Initializes the profile, then installs the two linked bundles. + dsh(["plugin", "--profile", "e2e", "install"]); + writeFileSync( + join(profileDir, "package.json"), + `${JSON.stringify( + { + dependencies: { + "dsh-host-probe": `link:${posix(relative(profileDir, PROBE_BUNDLE))}`, + "dsh-opencode-patch": `link:${posix(relative(profileDir, CHECKOUT))}`, + }, + dsh: { + profile: { + bundles: [ + "@deepseek-ai/dsh-base", + "@deepseek-ai/dsh-web-app", + "dsh-opencode-patch", + "dsh-host-probe", + ], + }, + }, + name: "dsh-profile-e2e", + private: true, + }, + null, + 2 + )}\n` + ); + dsh(["plugin", "--profile", "e2e", "install"]); + // The characterization profile, as JSON text (valid YAML). + writeFileSync( + join(profileDir, "cordis.patch.yml"), + `${JSON.stringify([{ config: { providers: CHARACTERIZED_PROFILE }, id: "llm-pi-ai" }], null, 2)}\n` + ); + // Dummy keys in the credentials store, never the environment: the plugin + // reads the environment directly, and the in-process rows were measured + // with the keys in the credentials service only. + mkdirSync(dshHome, { recursive: true }); + writeFileSync( + join(dshHome, ".credentials.yaml"), + "version: 1\nrefs:\n OPENCODE_API_KEY: probe-zen-not-a-key\n OPENCODE_GO_API_KEY: probe-go-not-a-key\n", + { mode: 0o600 } + ); + + const probeOut = join(home, "probe.json"); + const child = spawn( + process.execPath, + [bin, "--profile", "e2e", "--no-open", "--port", "0"], + { + cwd: home, + env: { + ...env, + PROBE_MODELS: JSON.stringify(CHARACTERIZED_MODELS), + PROBE_OUT: probeOut, + PROBE_PI_PACKAGE: PI_AI_PACKAGE, + PROBE_ROUTES: JSON.stringify(INTERNAL_ROUTES), + PROBE_SAVE: SAVE_CANDIDATE, + PROBE_SAVE_DROP: JSON.stringify(ZEN_UNDESCRIBED), + }, + stdio: ["ignore", "pipe", "pipe"], + } + ); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr = `${stderr}${chunk.toString("utf8")}`.slice(-4000); + }); + child.stdout.resume(); + const end = await new Promise((resolve) => { + const timer = setTimeout(() => { + child.kill("SIGKILL"); + resolve({ code: null, signal: "SIGKILL", timedOut: true }); + }, BOOT_DEADLINE_MS); + child.on("exit", (code, signal) => { + clearTimeout(timer); + resolve({ code, signal, timedOut: false }); + }); + }); + // Whatever the probe wrote is kept BEFORE the exit code is judged: a + // failed run is the one whose artifact matters. The probe writes URLs, + // model ids, verdicts and host error messages (cut to 500 characters) — + // never a header or a key value. + const written = existsSync(probeOut) + ? readFileSync(probeOut, "utf8") + : undefined; + const copy = process.env.DSH_HOST_PROBE_COPY ?? ""; + if (written !== undefined && copy.length > 0) { + writeFileSync(copy, written); + } + let parsed: ProbeOutput | undefined; + try { + parsed = + written === undefined + ? undefined + : (JSON.parse(written) as ProbeOutput); + } catch { + parsed = undefined; + } + if (end.code !== 0 || parsed === undefined || parsed.fatal !== undefined) { + throw new Error( + `${whyRunFailed(end, written, parsed)}\n--- stderr (tail) ---\n${stderr.slice(-2000)}` + ); + } + out = parsed; + console.info(`[e2e-host] probe output:\n${JSON.stringify(out, null, 2)}`); + }, BOOT_DEADLINE_MS * 2); + + afterAll(() => { + if (home.length > 0) { + rmSync(home, { force: true, recursive: true }); + } + }); + + it("measured with the probe innermost and the hidden routes settled", () => { + expect(out.fatal).toBeUndefined(); + expect(out.fetchInnermost).toBe(true); + expect(out.routesSettled).toBe(true); + }); + + it("[INTENDED_CHANGE] registry: the planes are pi-ai's; the hidden routes are registered and not listed (P-R1 claims the planes)", () => { + expect(out.providers).toEqual( + expect.arrayContaining(["opencode", "opencode-go"]) + ); + for (const route of INTERNAL_ROUTES) { + expect(out.providers).not.toContain(route); + expect(out.registryRoutes).toContain(route); + } + expect(out.owners.opencode).toBe("PiAiAdapter"); + expect(out.owners["opencode-go"]).toBe("PiAiAdapter"); + for (const plane of ["opencode", "opencode-go"]) { + expect(out.configurable).toContainEqual( + expect.objectContaining({ provider: plane, settingsNs: "llm-pi-ai" }) + ); + } + }); + + it("[INTENDED_CHANGE] keyless listing: every paid model is hidden before a key is captured", () => { + expect(out.listModels).toEqual(S0_TODAY.keylessListing); + }); + + // These two depend on which ids pi describes, so they are pinned for the + // pi this repository pins; the floating cell skips them and its probe JSON + // is the record. + describe("rows that depend on which ids pi describes", () => { + it.skipIf(!PINNED)( + "[KNOWN_DEFECT] keyed listing: the gap ids are missing (pinned pi)", + () => { + expect(out.listModelsAfter).toEqual(S0_TODAY.keyedListing); + } + ); + + it.skipIf(!PINNED)( + "resolveModelInfo, resolveCallConfig and the wire match the in-process rows for the decoration state the probe recorded (pinned pi)", + () => { + // The loader starts rows in parallel (F17), so whether the plugin's + // decoration wrapped pi-ai's plane adapter is a race: measured on the + // published binary it did not (the plugin applied about 70 ms first), + // which is no margin to pin. So the rows follow the recorded state, and + // `todayRowsFor` throws on a missing or unknown value. + console.info( + `[e2e-host] plane adapter decorated: ${String(out.planeAdapterDecorated)} (inferred order ${out.inferredOrder})` + ); + for (const [key, row] of Object.entries( + todayRowsFor(out.inferredOrder) + )) { + expect(out.modelInfo[key], key).toEqual(row.info); + expect(out.callConfig[key], key).toBe(row.callConfig); + expect(out.dispatch[key], key).toEqual({ + finish: row.finish, + sent: row.sent, + }); + } + } + ); + }); + + it("[BASELINE] the settings page reads every plane the profile declares", () => { + expect(out.describe?.value.toSorted()).toEqual([ + "deepseek", + "opencode", + "opencode-go", + ]); + }); + + it("[KNOWN_DEFECT] a stored gap id blocks saving any new id on its plane", () => { + // pi-ai's strict check names the FIRST stored id the installed pi cannot + // describe. The pinned pi lacks gpt-5.3-codex-spark, so it is named; a + // floating pi may have learned it, and then names the next undescribed id + // — so the floating cell asserts only that one of them blocks the save. + expect(out.saveBlocked.rejected).toMatch( + PINNED ? S0_TODAY.saveBlockedByStoredGap : SAVE_BLOCKED_BY_ANY_UNDESCRIBED + ); + }); + + it("[KNOWN_DEFECT] a catalog-unknown id cannot be saved, once the plane stores no gap id", () => { + // Clearing only ZEN_UNDESCRIBED assumes the installed pi still describes + // the other opencode ids the profile declares. The floating cell asserts + // it too: a pi that retires one of them would fail here, and that is a + // retirement worth a red nightly rather than a row to skip. + expect(out.saveCleared).toEqual({ + accepted: true, + dropped: [...ZEN_UNDESCRIBED], + }); + expect(out.save.rejected).toMatch(S0_TODAY.saveRejectedUnknown); + expect(out.afterSave).toBeUndefined(); + }); +}); diff --git a/test/e2e/host/cordis.patch.yml b/test/e2e/host/cordis.patch.yml new file mode 100644 index 00000000..e6df604d --- /dev/null +++ b/test/e2e/host/cordis.patch.yml @@ -0,0 +1,5 @@ +# The measurement bundle the e2e-host job links into a throwaway profile: one +# row, the probe. See test/e2e/host-binary.e2e.ts. +- insert: + - id: dsh-host-probe + name: dsh-host-probe diff --git a/test/e2e/host/package.json b/test/e2e/host/package.json new file mode 100644 index 00000000..b20a1677 --- /dev/null +++ b/test/e2e/host/package.json @@ -0,0 +1,12 @@ +{ + "name": "dsh-host-probe", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "probe-plugin.mjs", + "dsh": { + "bundle": { + "patch": "./cordis.patch.yml" + } + } +} diff --git a/test/e2e/host/probe-plugin.mjs b/test/e2e/host/probe-plugin.mjs new file mode 100644 index 00000000..163a07e3 --- /dev/null +++ b/test/e2e/host/probe-plugin.mjs @@ -0,0 +1,347 @@ +/** + * The measurement probe the `e2e-host` job boots inside the real host. + * + * A bundle plugin with one job: once the published `dsh` binary has booted a + * profile holding this plugin and `dsh-opencode-patch`, ask the host what it + * now serves — the registry, the listings, the validation verdicts, where one + * turn per model is sent, what the settings page reads, and whether a + * catalog-unknown id can be saved — write that as JSON to `PROBE_OUT`, and exit + * the process. `test/e2e/host-binary.e2e.ts` reads the file. + * + * It never reaches the network: `fetch` is replaced at module load, so every + * request to `opencode.ai` or `models.opencode.ai` is recorded (URL and model + * only — never a header, so never a key) and answered with a 401, and every + * other host passes through untouched. What it writes is URLs, model ids, + * verdicts and host error messages — never a header or a key value — and + * every error message is cut to {@link MESSAGE_LIMIT} characters, since one + * could name a path on the runner. + * + * Inputs, all from the environment the e2e sets: + * + * - `PROBE_OUT`: where to write the JSON; + * - `PROBE_MODELS`: `{ "": ["", …] }`, the ids to ask about; + * - `PROBE_SAVE`: an id to append to `opencode`'s models through the config + * editor, the way the Settings page saves one. It is tried three times over: + * onto the plane as booted (`saveBlocked`), then — after the ids in + * `PROBE_SAVE_DROP` are removed from the plane (`saveCleared`) — once more + * (`save`), so the last answer is about the id itself and not about a gap id + * the plane already stores; + * - `PROBE_SAVE_DROP`: `["", …]`, the ids to remove first; + * - `PROBE_ROUTES`: `["", …]`, the hidden routes to wait for, which the + * e2e passes from `INTERNAL_ROUTES` so the probe holds no copy of them; + * - `PROBE_PI_PACKAGE`: the package name pi-ai's loader entry carries; + * - `PROBE_SETTLE_MS`: how long to wait for the plugin's hidden routes + * (default 30000). + * + * @module test/e2e/host/probe-plugin + */ +import { writeFileSync } from "node:fs"; + +export const name = "dsh-host-probe"; + +const OUT = process.env.PROBE_OUT; +const SETTLE_MS = Number(process.env.PROBE_SETTLE_MS ?? "30000"); +const GATEWAY_HOSTS = new Set(["opencode.ai", "models.opencode.ai"]); +/** The routes today's plugin mounts behind the planes. */ +const HIDDEN_ROUTES = JSON.parse(process.env.PROBE_ROUTES ?? "[]"); +const PI_AI_PACKAGE = process.env.PROBE_PI_PACKAGE ?? ""; +/** The longest error message written: enough to triage, too short for a dump. */ +const MESSAGE_LIMIT = 500; + +/** An error's message, cut to {@link MESSAGE_LIMIT}. */ +const messageOf = (error) => + String(error?.message ?? error).slice(0, MESSAGE_LIMIT); + +/** A service the measurement needs, or a named failure where it is missing. */ +const required = (ctx, service) => { + const value = ctx.get(service); + if (value === undefined) { + throw new Error(`probe: no ${service} service in this host`); + } + return value; +}; + +const requests = []; +const passThrough = globalThis.fetch; + +const urlOf = (input) => { + if (typeof input === "string") return input; + return input instanceof URL ? input.href : input.url; +}; + +const hostOf = (url) => { + try { + return new URL(url).hostname; + } catch { + return ""; + } +}; + +const modelOf = async (input, init) => { + let body = init?.body; + if (body === undefined && input instanceof Request) { + body = await input.clone().text(); + } + try { + const { model } = JSON.parse(String(body)); + return typeof model === "string" ? model : null; + } catch { + return null; + } +}; + +const probeFetch = async (input, init) => { + const url = urlOf(input); + if (!GATEWAY_HOSTS.has(hostOf(url))) { + return passThrough(input, init); + } + requests.push({ model: await modelOf(input, init), url }); + return Response.json( + { error: { message: "probe: offline", type: "AuthError" } }, + { status: 401 } + ); +}; +// Installed at module load, which the host runs before any plugin's `apply` +// that needs a service: the plugin's fetch patch then wraps THIS function, so +// the probe sees the request the patch actually sends. +globalThis.fetch = probeFetch; + +/** A call's answer, or its failure as `{ code, message }`. */ +const settle = async (run) => { + try { + return { ok: await run() }; + } catch (error) { + return { + error: { + code: typeof error?.code === "string" ? error.code : undefined, + message: messageOf(error), + }, + }; + } +}; + +const sleep = (ms) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +/** Drain one turn through the host's own stream path. */ +const turn = async (llm, provider, model) => { + const before = requests.length; + let finish; + const drained = await settle(async () => { + for await (const chunk of llm.stream({ + messages: [{ content: [{ text: "hi", type: "text" }], role: "user" }], + model, + provider, + signal: AbortSignal.timeout(15_000), + })) { + if (chunk.type === "finish") { + finish = chunk.reason?.failure?.code ?? chunk.reason?.kind; + } + } + }); + return { + finish, + sent: requests + .slice(before) + .map((request) => request.url) + .filter((url) => hostOf(url) === "opencode.ai"), + ...(drained.error === undefined ? {} : { error: drained.error }), + }; +}; + +/** Every id in `PROBE_MODELS` asked three questions: listing, info, verdict. */ +const askAbout = async (llm, ids) => { + const listModels = {}; + const modelInfo = {}; + const callConfig = {}; + for (const [route, models] of Object.entries(ids)) { + const listed = await settle(() => llm.listModels(route)); + listModels[route] = listed.ok?.map((model) => model.id) ?? listed; + for (const model of models) { + const info = await settle(() => llm.resolveModelInfo(route, model)); + modelInfo[`${route}/${model}`] = info.ok + ? { + context: info.ok.context?.contextWindow, + efforts: + info.ok.reasoning?.efforts?.map((effort) => effort.id) ?? [], + } + : { error: info.error.code }; + const call = await settle(() => + llm.resolveCallConfig({ model, provider: route }) + ); + callConfig[`${route}/${model}`] = call.ok ? "ok" : call.error.code; + } + } + return { callConfig, listModels, modelInfo }; +}; + +/** One config-editor edit of `opencode`, as the Settings page makes it. */ +const editZen = async (editor, entry, mutate) => { + const saved = await settle(() => + editor.edit(entry, (current) => { + const providers = { ...current.providers }; + providers.opencode = mutate(providers.opencode ?? {}); + return { ...current, providers }; + }) + ); + return saved.error === undefined + ? { accepted: true } + : { rejected: saved.error.message }; +}; + +/** + * Save `saveId` into `opencode`'s models: onto the plane as booted, then onto + * the plane without the `drop` ids, so the last verdict is about `saveId`. + */ +const saveSequence = async (llm, editor, saveId, drop) => { + const entry = editor + .entries() + .find((candidate) => candidate.options?.name === PI_AI_PACKAGE); + const append = (plane) => ({ + ...plane, + models: [...(plane.models ?? []), { id: saveId }], + }); + const blocked = await editZen(editor, entry, append); + let out = { saveBlocked: { id: saveId, ...blocked } }; + let saved = blocked; + if (blocked.rejected !== undefined) { + const cleared = await editZen(editor, entry, (plane) => ({ + ...plane, + models: (plane.models ?? []).filter((model) => !drop.includes(model.id)), + })); + saved = await editZen(editor, entry, append); + out = { ...out, saveCleared: { dropped: drop, ...cleared } }; + } + out = { ...out, save: { id: saveId, ...saved } }; + if (saved.rejected !== undefined) { + return out; + } + const listed = await settle(() => llm.listModels("opencode")); + const call = await settle(() => + llm.resolveCallConfig({ model: saveId, provider: "opencode" }) + ); + return { + ...out, + afterSave: { + callConfig: call.error?.code ?? "ok", + dispatch: await turn(llm, "opencode", saveId), + listed: listed.ok?.map((model) => model.id) ?? listed, + }, + }; +}; + +const measure = async (ctx) => { + if (HIDDEN_ROUTES.length === 0 || PI_AI_PACKAGE.length === 0) { + throw new Error("probe: PROBE_ROUTES and PROBE_PI_PACKAGE must be set"); + } + await ctx.get("loader")?.await?.(); + const llm = required(ctx, "llm"); + const registry = () => llm.adapters; + const until = Date.now() + SETTLE_MS; + while ( + HIDDEN_ROUTES.some((route) => !registry().has(route)) && + Date.now() < until + ) { + await sleep(100); + } + const ids = JSON.parse(process.env.PROBE_MODELS ?? "{}"); + const registered = { + configurable: llm + .listConfigurableProviders() + .filter((entry) => /opencode/u.test(entry.provider)) + .map((entry) => ({ + provider: entry.provider, + settingsNs: entry.settingsNs, + settingsPath: entry.settingsPath, + })), + fetchInnermost: globalThis.fetch !== probeFetch, + owners: Object.fromEntries( + [...registry()].map(([route, row]) => [ + route, + row.adapter.constructor.name, + ]) + ), + // Whether the plugin's adapter decoration (decoratePlaneAdapters in + // src/models-discovery.ts) wrapped pi-ai's plane adapter. It wraps only + // the adapters registered when its effect ran, so the apply order is + // INFERRED from it (`inferredOrder`) — not observed: a pi-ai restart after + // the plugin applied would read as plugin-first. The per-model rows depend + // on the decoration, not on the order as such, so this is what they need. + planeAdapterDecorated: Object.hasOwn( + registry().get("opencode")?.adapter ?? {}, + "resolveModel" + ), + providers: llm.listProviders().map((provider) => provider.id), + registryRoutes: [...registry().keys()], + routesSettled: HIDDEN_ROUTES.every((route) => registry().has(route)), + }; + registered.inferredOrder = registered.planeAdapterDecorated + ? "pi-ai-first" + : "plugin-first"; + const asked = await askAbout(llm, ids); + const zenNs = registered.configurable.find( + (entry) => entry.provider === "opencode" + )?.settingsNs; + const discovered = await settle(() => + llm.discoverModels(zenNs, { provider: "opencode" }) + ); + const dispatch = {}; + for (const [route, models] of Object.entries(ids)) { + for (const model of models) { + dispatch[`${route}/${model}`] = await turn(llm, route, model); + } + } + const listModelsAfter = {}; + for (const route of Object.keys(ids)) { + const listed = await settle(() => llm.listModels(route)); + listModelsAfter[route] = listed.ok?.map((model) => model.id) ?? listed; + } + const piDescriptor = required(ctx, "settings") + .describe() + .find((descriptor) => descriptor.ns === "llm-pi-ai"); + const editor = required(ctx, "configEditor"); + const saveId = process.env.PROBE_SAVE ?? ""; + const drop = JSON.parse(process.env.PROBE_SAVE_DROP ?? "[]"); + const saving = + saveId.length === 0 ? {} : await saveSequence(llm, editor, saveId, drop); + return { + ...registered, + ...asked, + ...saving, + describe: + piDescriptor === undefined + ? undefined + : { + user: Object.keys(piDescriptor.user?.providers ?? {}), + value: Object.keys(piDescriptor.value?.providers ?? {}), + }, + discovery: discovered.ok ? { count: discovered.ok.length } : discovered, + dispatch, + listModelsAfter, + }; +}; + +export const apply = (ctx) => { + let stopped = false; + ctx.effect?.(() => () => { + stopped = true; + }); + // Not awaited: `apply` returns at once, and the measurement runs once the + // loader has settled every row. + void (async () => { + try { + const out = await measure(ctx); + if (stopped) return; + writeFileSync(OUT, `${JSON.stringify(out, null, 2)}\n`); + setTimeout(() => process.exit(0), 200); + } catch (error) { + const fatal = messageOf(error); + // On stderr too: the e2e shows the tail when the host exits non-zero. + process.stderr.write(`[dsh-host-probe] fatal: ${fatal}\n`); + writeFileSync(OUT, `${JSON.stringify({ fatal })}\n`); + setTimeout(() => process.exit(1), 200); + } + })(); +}; diff --git a/test/e2e/keyless-matrix.e2e.ts b/test/e2e/keyless-matrix.e2e.ts new file mode 100644 index 00000000..c803deef --- /dev/null +++ b/test/e2e/keyless-matrix.e2e.ts @@ -0,0 +1,185 @@ +/** + * The keyless 401-vs-404 matrix against the live gateway. + * + * Sixteen POSTs, none carrying a key: two planes × four protocols × the + * legacy (`/zen(/go)/v1`) and modern (`/inference(/go)/…`) endpoint families, + * each with one PAID model from the live catalog slice (a free model answers + * keyless with a real completion, so the matrix never picks one). The gateway + * authenticates before it routes, so the answer says whether the PATH exists + * without spending anything. It runs on every pull request, because it needs + * no secret; it never reads `OPENCODE_API_KEY` or `OPENCODE_GO_API_KEY`. + * + * What it asserts: + * + * - every one of the sixteen cells has a model and was SENT, and every answer + * is one the matrix can read — an empty or reshaped catalog fails here + * rather than measuring nothing; + * - no cell today's plugin dispatches to with the default endpoint (`auto`, + * an alias of `modern`; `TODAY_DISPATCH`: completions on the legacy family, + * the hidden routes' Responses, Messages and Google on the modern one) + * answers "path gone"; + * - nor do the modern completions cells (`NEXT_DISPATCH`), where P-R1 moves + * completions: a forward guard, labelled as such when it fails. + * + * The other legacy verdicts are recorded, not asserted: P-R2 decides the + * family from them. They include the cells an `endpoint: legacy` user + * dispatches to — Responses and Messages on both planes, and Zen's Google — + * so this file does not guard that configuration; P-R2 decides whether it + * should. Go's Google stays on `/inference/go/google/v1beta` under every + * family (`GO_GOOGLE_BASE_URL`), the modern cell `TODAY_DISPATCH` asserts. + * The cell table, both sets and the classifier are unit-tested offline in + * `test/keyless-matrix.test.ts`. + * + * It always measures the vendor's own gateway: the cells are the vendor's + * paths, which a mirror's `/zen(/go)/v1` base cannot name. So it SKIPS when + * `OPENCODE_ZEN_BASE_URL` or `OPENCODE_GO_BASE_URL` points the rest of the + * suite at a mirror. + * + * Opt-in: `OPENCODE_E2E=1 pnpm run test:e2e`. + * + * @module test/e2e/keyless-matrix.e2e + */ +import { describe, expect, it } from "vitest"; + +import { + cellKey, + classifyKeylessProbe, + isTransientProbeAnswer, + type KeylessVerdict, + keylessMatrix, + type MatrixCell, + NEXT_DISPATCH, + TODAY_DISPATCH, +} from "../support/keyless-matrix.ts"; +import { MODELS_OPENCODE_URL, parseModelsDevCatalog } from "../support/src.ts"; + +const LIVE = process.env.OPENCODE_E2E === "1"; +/** A mirror the rest of the suite is pointed at, which this file cannot follow. */ +const MIRRORED = ["OPENCODE_ZEN_BASE_URL", "OPENCODE_GO_BASE_URL"].some( + (name) => (process.env[name] ?? "").length > 0 +); +if (LIVE && MIRRORED) { + console.info( + "[keyless-matrix] skipped: a mirror base URL is set, and the matrix measures only the vendor's own paths" + ); +} + +const PROBE_TIMEOUT_MS = 30_000; +/** The pause before the one retry of a transient answer. */ +const RETRY_AFTER_MS = 2000; + +/** One probe's outcome, with no request or response header in it. */ +interface ProbeRow { + readonly cell: string; + readonly model: string | undefined; + readonly status: number | undefined; + readonly verdict: KeylessVerdict | "transport-error"; + /** The start of the body, kept only for a verdict the matrix cannot read. */ + readonly body?: string; +} + +/** One keyless request: its status and body, or `undefined` on a transport error. */ +const send = async ( + cell: MatrixCell +): Promise<{ status: number; bodyText: string } | undefined> => { + try { + const response = await fetch(cell.url, { + body: JSON.stringify(cell.body), + headers: cell.headers, + method: cell.method, + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + return { bodyText: await response.text(), status: response.status }; + } catch { + return undefined; + } +}; + +const probe = async (cell: MatrixCell): Promise => { + const name = cellKey(cell); + if (cell.body === undefined) { + return { + cell: name, + model: undefined, + status: undefined, + verdict: "no-model", + }; + } + let answer = await send(cell); + // A rate limit, a server error or a dropped connection is retried once + // before it is read: one transient answer must not fail every pull request. + if (isTransientProbeAnswer(answer?.status)) { + await new Promise((resolve) => { + setTimeout(resolve, RETRY_AFTER_MS); + }); + answer = await send(cell); + } + if (answer === undefined) { + return { + cell: name, + model: cell.model, + status: undefined, + verdict: "transport-error", + }; + } + const verdict = classifyKeylessProbe(answer); + return { + cell: name, + model: cell.model, + status: answer.status, + verdict, + ...(verdict === "other" ? { body: answer.bodyText.slice(0, 200) } : {}), + }; +}; + +describe.skipIf(!LIVE || MIRRORED)( + "live gateway · the keyless path matrix", + () => { + it( + "sends every cell, reads every answer, and no cell the plugin dispatches to (or P-R1 will) answers path-gone", + async () => { + const listing = await fetch(MODELS_OPENCODE_URL, { + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + expect(listing.status).toBe(200); + const parsed = parseModelsDevCatalog(await listing.json()); + // The parsed slices have already dropped their deprecated rows, so no + // `isDeprecated` predicate is passed: it could never exclude an id here. + const cells = keylessMatrix({ go: parsed.go, zen: parsed.zen }); + expect(cells).toHaveLength(16); + const rows: ProbeRow[] = []; + for (const cell of cells) { + rows.push(await probe(cell)); + } + console.table(rows); + // A model-less cell sends nothing. On the bundled catalogs every cell + // has one, so a live slice without one is a catalog change to triage — + // a reshaped listing would otherwise pass after sending no probe at all. + const unsent = rows.filter((row) => row.verdict === "no-model"); + expect(unsent, JSON.stringify(unsent)).toEqual([]); + // A 200 stays unreadable on purpose: it means a free model slipped past + // the matrix's filter and a completion was spent. + const unread = rows.filter( + (row) => + !["path-exists", "model-gone", "path-gone"].includes(row.verdict) + ); + expect(unread, JSON.stringify(unread)).toEqual([]); + const goneToday = rows.filter( + (row) => TODAY_DISPATCH.has(row.cell) && row.verdict === "path-gone" + ); + expect( + goneToday, + `a path today's plugin dispatches to is gone: ${JSON.stringify(goneToday)}` + ).toEqual([]); + const goneNext = rows.filter( + (row) => NEXT_DISPATCH.has(row.cell) && row.verdict === "path-gone" + ); + expect( + goneNext, + `forward guard: a path P-R1 moves completions to is gone: ${JSON.stringify(goneNext)}` + ).toEqual([]); + }, + 16 * 2 * (PROBE_TIMEOUT_MS + RETRY_AFTER_MS) + PROBE_TIMEOUT_MS + ); + } +); diff --git a/test/host-characterization.test.ts b/test/host-characterization.test.ts new file mode 100644 index 00000000..70d96a3c --- /dev/null +++ b/test/host-characterization.test.ts @@ -0,0 +1,707 @@ +/** + * TODAY's plugin on the real host, characterized (tier B, pi 1.1.0). + * + * The plugin is mounted as it ships — its row carries no config, so the + * shipped defaults apply: the hidden ids in `providers`, `enrichModels` on, + * `endpoint: auto` — beside pi-ai holding the characterization profile (S0 + * plus a declared catalog-unknown id on each plane) with a key stored for each + * plane. Every row asserts what was MEASURED, and is tagged so a later phase + * flips named rows (docs/improvement-plan.md, "Phases"): + * + * - `[KNOWN_DEFECT]` is wrong today, and a phase fixes it; + * - `[INTENDED_CHANGE]` is right today, and a phase changes it on purpose; + * - `[BASELINE]` must survive every phase. + * + * The per-model expectations live in `S0_TODAY` (`support/host.ts`), which + * the binary e2e reads too, so the in-process host and the published binary + * cannot drift apart unnoticed. Each model's three facets (model info, the + * call-config verdict, the wire) carry their own tag there, and this file + * generates one test per model and facet titled with that tag, so the tag is + * what a reader and a phase's diff go by. Where a measurement contradicted the + * plan's expectation the row keeps the measurement and says so. + * + * Every case boots its own host, so a key the fetch patch captured in one case + * cannot change another's listing. The main block boots in the published + * host's apply order (`plugin-first`: the plugin applies before pi-ai + * registers, measured on the binary); a second block boots `pi-ai-first` and + * records the rows that change with the order — the adapter decoration only + * wraps what was registered when the plugin applied. + * + * @module test/host-characterization.test + */ +import { readFileSync } from "node:fs"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + adaptersOf, + type ApplyOrder, + APPLY_ORDERS, + bootTierB, + CHARACTERIZED_MODELS, + characterizedProviders, + drainStream, + fakeCredentials, + PLANES, + piKnows, + providerWithApiKey, + recordingFetch, + type RecordingFetch, + type RowFacet, + S0_TODAY, + s0Providers, + SAVE_BLOCKED_BY_ANY_UNDESCRIBED, + SAVE_CANDIDATE, + type TierB, + todayRowsFor, + type TodayRow, + UNKNOWN_ID, + waitForRoutes, + ZEN_UNDESCRIBED, +} from "./support/host.ts"; +import { + INTERNAL_ROUTES, + OPENCODE_GO_CATALOG, + OPENCODE_ZEN_CATALOG, + PI_THINKING_LEVELS, + thinkingLevelMapFor, +} from "./support/src.ts"; + +type RowKey = keyof typeof S0_TODAY.rows; + +let tier: TierB | undefined; +let wire: RecordingFetch; + +/** Boot the characterization profile with a key stored for each plane, the plugin as it ships. */ +const bootS0 = async (order: ApplyOrder): Promise => { + wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + tier = await bootTierB({ + credentials: fakeCredentials({ + refs: { + OPENCODE_API_KEY: "zen-test-key", + OPENCODE_GO_API_KEY: "go-test-key", + }, + }), + order, + providers: characterizedProviders(), + }); + // The plugin mounts its hidden routes asynchronously (mountInternalRoutes). + await waitForRoutes(tier.llm, INTERNAL_ROUTES); +}; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; + vi.unstubAllGlobals(); +}); + +const booted = (): TierB => { + if (tier === undefined) { + throw new Error("no host booted"); + } + return tier; +}; + +/** One turn's gateway URLs, the catalog read excluded. */ +const sentBy = async (route: string, model: string) => { + const before = wire.requests.length; + const drained = await drainStream(booted().llm, route, model); + return { + drained, + requests: wire.requests + .slice(before) + .filter( + (request) => !request.url.startsWith("https://models.opencode.ai/") + ), + }; +}; + +/** pi-ai's plane adapter, private methods included. */ +const planeAdapter = () => + adaptersOf(booted().llm).get("opencode")?.adapter as unknown as { + prepareCall: ( + route: string, + model: string + ) => Promise<{ baseUrl?: string; api?: string }>; + }; + +/** `resolveCallConfig`'s verdict: "ok", or the code it rejected with. */ +const callConfigOf = async (route: string, model: string): Promise => { + try { + await booted().llm.resolveCallConfig({ model, provider: route } as never); + return "ok"; + } catch (error) { + return String((error as { code?: unknown }).code); + } +}; + +const ROW_KEYS = Object.keys(S0_TODAY.rows) as RowKey[]; +const split = (key: RowKey): [string, string] => { + const [route = "", ...rest] = key.split("/"); + return [route, rest.join("/")]; +}; + +/** One facet of one row, measured on the booted host. */ +const measureFacet = async (facet: RowFacet, key: RowKey): Promise => { + const [route, model] = split(key); + if (facet === "info") { + try { + const info = await booted().llm.resolveModelInfo(route, model); + return { + context: info.context?.contextWindow, + efforts: info.reasoning?.efforts?.map((effort) => effort.id) ?? [], + }; + } catch (error) { + return { error: String((error as { code?: unknown }).code) }; + } + } + if (facet === "callConfig") { + return callConfigOf(route, model); + } + const { drained, requests } = await sentBy(route, model); + // Every body-addressed request carries the model it was for. + for (const request of requests) { + if (!request.url.includes(":streamGenerateContent")) { + expect(request.model).toBe(model); + } + } + return { + finish: drained.finish?.code, + sent: requests.map((request) => request.url), + }; +}; + +/** What `S0_TODAY` (or another order's table) records for one facet of one row. */ +const expectedFacet = ( + facet: RowFacet, + key: RowKey, + rows: Readonly> = S0_TODAY.rows +): unknown => { + const row = rows[key]; + if (facet === "info") { + return row.info; + } + if (facet === "callConfig") { + return row.callConfig; + } + return { finish: row.finish, sent: row.sent }; +}; + +/** A config-editor mutation appending `id` to a plane's models. */ +const appendTo = + (plane: string, id: string) => + (current: Record): Record => { + const providers = current.providers as Record< + string, + { models?: { id: string }[] } + >; + return { + ...current, + providers: { + ...providers, + [plane]: { + ...providers[plane], + models: [...(providers[plane]?.models ?? []), { id }], + }, + }, + }; + }; + +/** A config-editor mutation dropping `ids` from a plane's models. */ +const dropFrom = + (plane: string, ids: readonly string[]) => + (current: Record): Record => { + const providers = current.providers as Record< + string, + { models?: { id: string }[] } + >; + return { + ...current, + providers: { + ...providers, + [plane]: { + ...providers[plane], + models: (providers[plane]?.models ?? []).filter( + (model) => !ids.includes(model.id) + ), + }, + }, + }; + }; + +/** One generated case per row and facet, titled with the facet's own tag. */ +const FACET_CASES = ROW_KEYS.flatMap((key) => + (["info", "callConfig", "wire"] as const).map((facet) => ({ + facet, + key, + tag: S0_TODAY.rows[key].tags[facet], + })) +); + +/** The plane a row key belongs to, and the bundled catalog for it. */ +const catalogFor = (key: RowKey) => + key.startsWith("opencode-go/") ? OPENCODE_GO_CATALOG : OPENCODE_ZEN_CATALOG; + +/** The provider keys one side of a settings descriptor holds. */ +const providerKeys = (side: unknown): string[] => + Object.keys((side as { providers?: object } | undefined)?.providers ?? {}); + +/** + * The `[BASELINE]` session-controller reads after a Sign-in, on today's plugin + * with only the two plane keys stored (what `bootS0` stores): the Zen plane + * answers `hasProviderApiKey`, and pi-ai's `settings.describe` value lists + * both planes. P-R1 strips the planes from pi-ai's resolved config, which on + * the host alone turns this false (F20, the plugin-less contract in + * `test/host-editor-contracts.test.ts`); its D18 wrapper is what must keep + * this case green. + */ +const expectSignInReadsTheZenPlane = async (): Promise => { + const host = booted(); + const from = host.credentials.calls.length; + expect(await providerWithApiKey(host)).toBe("opencode"); + expect( + host.credentials.calls + .slice(from) + .filter(([method]) => method === "describe") + .map(([, ref]) => ref) + ).toContain("OPENCODE_API_KEY"); + const descriptor = host.ctx.settings + .describe() + .find((entry) => entry.ns === "llm-pi-ai"); + expect(providerKeys(descriptor?.value).toSorted()).toEqual([ + "deepseek", + ...PLANES, + ]); +}; + +describe( + "characterization · today's plugin on pi 1.1.0, the published host's order (tier B)", + { timeout: 30_000 }, + () => { + beforeEach(() => bootS0("plugin-first"), 30_000); + + it("fixture: the gap ids are gaps on pi 1.1.0, the unknown id is unknown everywhere, and the known ids are known", () => { + expect(piKnows("opencode", "gpt-5.3-codex-spark")).toBeUndefined(); + expect( + piKnows("opencode-go", "gemini-4-argon-contributor") + ).toBeUndefined(); + expect(piKnows("opencode-go", "minimax-m3.1-flash")).toBeUndefined(); + expect(piKnows("opencode", "deepseek-v4-pro")).toBe("openai-completions"); + expect(piKnows("opencode", "gemini-3-flash")).toBe( + "google-generative-ai" + ); + expect(piKnows("opencode-go", "minimax-m3")).toBe("anthropic-messages"); + // The gap ids are ones the plugin's catalog still describes; the + // declared unknown id is in neither catalog (G1's two classes). + expect( + OPENCODE_ZEN_CATALOG.some((model) => model.id === "gpt-5.3-codex-spark") + ).toBe(true); + for (const plane of PLANES) { + expect(piKnows(plane, UNKNOWN_ID)).toBeUndefined(); + expect(CHARACTERIZED_MODELS[plane].at(-1)).toBe(UNKNOWN_ID); + } + for (const catalog of [OPENCODE_ZEN_CATALOG, OPENCODE_GO_CATALOG]) { + expect(catalog.some((model) => model.id === UNKNOWN_ID)).toBe(false); + expect(catalog.some((model) => model.id === SAVE_CANDIDATE)).toBe( + false + ); + } + expect(ROW_KEYS).toEqual( + PLANES.flatMap((plane) => + CHARACTERIZED_MODELS[plane].map((id) => `${plane}/${id}`) + ) + ); + expect(INTERNAL_ROUTES).toHaveLength(6); + }); + + it("fixture: each row's context and efforts are the bundled catalog's, so a catalog refresh that moves a fixture model fails here by name", () => { + // S0_TODAY copies context windows AND effort lists from the generated + // catalog, which the nightly refresh rewrites: decorateModelInfo makes + // the catalog authoritative for both (src/models-discovery.ts, the + // `thinkingLevelMapFor` → effortsFromLevelMap derivation, re-stated + // here from its two exported halves). A refresh that changes one of + // these ids must update S0_TODAY (and with it the binary e2e) — and must + // fail HERE, by name, not as a baseline facet that seems to regress. + // + // Two rows' `info` facets are KNOWN_DEFECT today and still copy the + // catalog (Zen gpt-5.3-codex-spark's context, Go minimax-m3's efforts); + // the plan moves them OFF the catalog on purpose (D17: the user's row + // wins; D1: a toggle model offers Off/On). The phase that flips such a + // facet adds its key to INFO_NOT_FROM_CATALOG in the same diff — and only + // a non-BASELINE `info` facet may be exempted, since a BASELINE one is + // meant to stay the catalog's across phases. + const INFO_NOT_FROM_CATALOG = new Set(); + for (const key of INFO_NOT_FROM_CATALOG) { + expect(ROW_KEYS, `${key} is not a characterized row`).toContain(key); + expect( + S0_TODAY.rows[key].tags.info, + `${key}: a BASELINE info facet cannot be exempted from the catalog` + ).not.toBe("BASELINE"); + } + let compared = 0; + for (const key of ROW_KEYS) { + if (INFO_NOT_FROM_CATALOG.has(key)) { + continue; + } + const [, id] = split(key); + const spec = catalogFor(key).find((model) => model.id === id); + if (spec === undefined) { + expect(id, key).toBe(UNKNOWN_ID); + continue; + } + const stale = + `S0_TODAY's info for ${key} no longer matches the bundled catalog: ` + + "either a catalog refresh moved it (update the row in " + + "test/support/host.ts), or a phase flipped this facet off the " + + "catalog (add the key to INFO_NOT_FROM_CATALOG here)"; + const info = S0_TODAY.rows[key].info as { + context?: number; + efforts?: readonly string[]; + }; + expect(spec.context_window, stale).toEqual(info.context); + if (info.efforts === undefined) { + continue; + } + const map = thinkingLevelMapFor(spec); + expect( + map === undefined + ? [] + : PI_THINKING_LEVELS.filter((level) => map[level] !== null), + stale + ).toEqual(info.efforts); + compared += 1; + } + // The case must have compared something: an `info` shape change that + // drops `efforts` would otherwise pass vacuously. + expect(compared).toBeGreaterThan(0); + }); + + it("fixture: the binary e2e's cell-aware helpers — the floating save-block pattern names any undescribed id and only those, and todayRowsFor refuses an order it does not know", () => { + // The pinned message is one the floating pattern accepts, so the two + // cells cannot disagree on a pinned run. + const pinned = + 'provider "opencode" model "gpt-5.3-codex-spark" needs an api'; + expect(pinned).toMatch(S0_TODAY.saveBlockedByStoredGap); + expect(pinned).toMatch(SAVE_BLOCKED_BY_ANY_UNDESCRIBED); + // A floating pi that learned gpt-5.3-codex-spark names the next one. + expect(`provider "opencode" model "${UNKNOWN_ID}" needs an api`).toMatch( + SAVE_BLOCKED_BY_ANY_UNDESCRIBED + ); + // A described id, or the dot read as a wildcard, is not a match. + for (const other of ["deepseek-v4-pro", "gpt-5x3-codex-spark"]) { + expect(`provider "opencode" model "${other}" needs an api`).not.toMatch( + SAVE_BLOCKED_BY_ANY_UNDESCRIBED + ); + } + for (const order of APPLY_ORDERS) { + expect(Object.keys(todayRowsFor(order))).toEqual(ROW_KEYS); + } + // A probe JSON missing the field reads `undefined`; it must not select + // the pi-ai-first table. + expect(() => todayRowsFor(undefined as unknown as ApplyOrder)).toThrow( + /unknown apply order/u + ); + }); + + it("[BASELINE] hasProviderApiKey answers the Zen plane with only the plane keys stored, and llm-pi-ai's settings.describe value lists both planes (F20, D18)", async () => { + await expectSignInReadsTheZenPlane(); + }); + + it("[INTENDED_CHANGE] registry: both planes are pi-ai's, and six hidden routes are registered and hidden from listProviders (P-R1 claims the planes)", () => { + const { llm } = booted(); + const registry = adaptersOf(llm); + expect([...registry.keys()].toSorted()).toEqual( + ["deepseek", "opencode", "opencode-go", ...INTERNAL_ROUTES].toSorted() + ); + expect( + llm + .listProviders() + .map((provider) => provider.id) + .toSorted() + ).toEqual(["deepseek", "opencode", "opencode-go"]); + for (const route of [...PLANES, ...INTERNAL_ROUTES]) { + expect(registry.get(route)?.adapter.constructor.name).toBe( + "PiAiAdapter" + ); + } + // pi-ai's own adapter serves the planes; the hidden routes are not it. + const piAdapter = registry.get("opencode")?.adapter; + expect(registry.get("opencode-go")?.adapter).toBe(piAdapter); + for (const route of INTERNAL_ROUTES) { + expect(registry.get(route)?.adapter).not.toBe(piAdapter); + } + const directory = llm.listConfigurableProviders(); + for (const plane of PLANES) { + expect( + directory.find((entry) => entry.provider === plane) + ).toMatchObject({ + settingsNs: "llm-pi-ai", + settingsPath: ["providers", plane], + }); + } + // The directory reports each plane's first stored gap id. + expect( + directory.find((entry) => entry.provider === "opencode")?.error + ).toMatch(/model "gpt-5\.3-codex-spark" needs an api/u); + expect( + directory.find((entry) => entry.provider === "opencode-go")?.error + ).toMatch(/model "gemini-4-argon-contributor" needs an api/u); + }); + + it("[INTENDED_CHANGE] keyless listing: before a key is captured, the listing patch hides every paid model on both planes (D21)", async () => { + const { llm } = booted(); + for (const plane of PLANES) { + expect(await llm.listModels(plane)).toEqual( + S0_TODAY.keylessListing[plane] + ); + } + }); + + it("[KNOWN_DEFECT] keyed listing: once a turn captured a key, only pi-resolved ids are listed — every gap id and the declared unknown id are missing (G1)", async () => { + const { llm } = booted(); + await sentBy("opencode", "deepseek-v4-pro"); + await sentBy("opencode-go", "deepseek-v4-pro"); + const listed: Record = {}; + for (const plane of PLANES) { + const models = await llm.listModels(plane); + listed[plane] = models.map((model) => model.id); + } + expect(listed).toEqual(S0_TODAY.keyedListing); + expect(listed.opencode).not.toContain("gpt-5.3-codex-spark"); + expect(listed["opencode-go"]).not.toContain("gemini-4-argon-contributor"); + expect(listed["opencode-go"]).not.toContain("minimax-m3.1-flash"); + for (const plane of PLANES) { + expect(listed[plane]).not.toContain(UNKNOWN_ID); + } + }); + + it.each(FACET_CASES)("[$tag] $key · $facet", async ({ facet, key }) => { + expect(await measureFacet(facet, key)).toEqual(expectedFacet(facet, key)); + }); + + it("[KNOWN_DEFECT] the toggle-only model is offered Minimal/Xhigh/Max (D1)", async () => { + // Chosen from the bundled catalog, not by name: on/off reasoning, no levels. + const spec = OPENCODE_GO_CATALOG.find( + (model) => model.id === "minimax-m3" + ); + expect(spec?.reasoning_options).toEqual([{ type: "toggle" }]); + const info = await booted().llm.resolveModelInfo( + "opencode-go", + "minimax-m3" + ); + const efforts = info.reasoning?.efforts?.map((effort) => effort.id) ?? []; + expect(efforts).toEqual( + S0_TODAY.rows["opencode-go/minimax-m3"].info.efforts + ); + expect(efforts).toEqual( + expect.arrayContaining(["minimal", "xhigh", "max"]) + ); + expect(efforts).not.toContain("off"); + }); + + it("[INTENDED_CHANGE] completions models dispatch to pi 1.1.0's legacy bases", async () => { + for (const [route, wireUrl] of [ + ["opencode", "https://opencode.ai/zen/v1/chat/completions"], + ["opencode-go", "https://opencode.ai/zen/go/v1/chat/completions"], + ] as const) { + const { drained, requests } = await sentBy(route, "deepseek-v4-pro"); + expect(requests.map((request) => request.url)).toEqual([wireUrl]); + expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); + } + }); + + it("[KNOWN_DEFECT] in this order the adapter decoration never runs: pi-ai's plane adapter is not wrapped, so its prepared call carries no repaired base (decoratePlaneAdapters in src/models-discovery.ts)", async () => { + // decoratePlaneAdapters wraps the adapters the registry holds when the + // plugin's effect runs, and here pi-ai has not registered yet. + const adapter = planeAdapter(); + expect(Object.hasOwn(adapter, "prepareCall")).toBe(false); + expect(Object.hasOwn(adapter, "resolveModel")).toBe(false); + const prepared = await adapter.prepareCall("opencode", "deepseek-v4-pro"); + expect(prepared.baseUrl).toBeUndefined(); + }); + + it("[KNOWN_DEFECT] Zen gpt-5.3-codex-spark: the user's row is ignored and resolveCallConfig refuses what dispatch serves through the hidden Responses route", async () => { + const { llm } = booted(); + const info = await llm.resolveModelInfo( + "opencode", + "gpt-5.3-codex-spark" + ); + // The row declares name "My Spark" and contextWindow 400000. + expect(info.context?.contextWindow).toBe(128_000); + expect(info.name).not.toBe("My Spark"); + await expect( + llm.resolveCallConfig({ + model: "gpt-5.3-codex-spark", + provider: "opencode", + } as never) + ).rejects.toMatchObject({ code: "INVALID_CONFIG" }); + const { requests } = await sentBy("opencode", "gpt-5.3-codex-spark"); + expect(requests.map((request) => request.url)).toEqual([ + "https://opencode.ai/inference/openai/v1/responses", + ]); + }); + + it("[KNOWN_DEFECT] Go minimax-m3.1-flash answers INVALID_CONFIG 'needs an api' and sends nothing (F4)", async () => { + const { llm } = booted(); + await expect( + llm.resolveCallConfig({ + model: "minimax-m3.1-flash", + provider: "opencode-go", + } as never) + ).rejects.toMatchObject({ + code: "INVALID_CONFIG", + message: expect.stringMatching(/"minimax-m3\.1-flash" needs an api/u), + }); + const { drained, requests } = await sentBy( + "opencode-go", + "minimax-m3.1-flash" + ); + expect(requests).toEqual([]); + expect(drained.finish).toEqual({ code: "INVALID_CONFIG", kind: "error" }); + }); + + it("[KNOWN_DEFECT] Go gemini-4-argon-contributor: resolveCallConfig refuses it (INVALID_CONFIG) while the turn is served on the Go Google base through the bridge", async () => { + expect( + await callConfigOf("opencode-go", "gemini-4-argon-contributor") + ).toBe("INVALID_CONFIG"); + const { drained, requests } = await sentBy( + "opencode-go", + "gemini-4-argon-contributor" + ); + expect(requests.map((request) => request.url)).toEqual( + S0_TODAY.rows["opencode-go/gemini-4-argon-contributor"].sent + ); + expect(requests[0]?.headers["x-goog-api-key"]).toBe("go-test-key".length); + // pi maps a Google 401 to its own code, not AUTH (as with pi alone). + expect(drained.finish).toEqual({ code: "PI_AI_ERROR", kind: "error" }); + }); + + it("[KNOWN_DEFECT] a stored gap id blocks every save of its plane: appending a new id and a displayName change are both refused naming the STORED id, and the file is unchanged (F4; llm-pi-ai config.ts:364-368, catalog.ts:917-921)", async () => { + const { ctx, piEntry, profile } = booted(); + const before = readFileSync(profile.patchPath); + await expect( + ctx.configEditor.edit(piEntry, appendTo("opencode", SAVE_CANDIDATE)) + ).rejects.toThrow(S0_TODAY.saveBlockedByStoredGap); + await expect( + ctx.configEditor.edit(piEntry, (current) => { + const providers = current.providers as Record; + return { + ...current, + providers: { + ...providers, + opencode: { ...providers.opencode, displayName: "Zen (edited)" }, + }, + }; + }) + ).rejects.toThrow(S0_TODAY.saveBlockedByStoredGap); + expect(readFileSync(profile.patchPath)).toEqual(before); + }); + + it("[KNOWN_DEFECT] the probe's save sequence: dropping the plane's undescribed ids is accepted, then the catalog-unknown id itself is what the check names", async () => { + // The binary e2e's probe runs exactly this, so its rows isolate the + // unknown id although every plane of the profile stores a gap id. + const { ctx, piEntry, profile } = booted(); + await ctx.configEditor.edit( + piEntry, + dropFrom("opencode", ZEN_UNDESCRIBED) + ); + const cleared = readFileSync(profile.patchPath, "utf8"); + expect(cleared).not.toContain("gpt-5.3-codex-spark"); + expect(cleared).toContain("gemini-4-argon-contributor"); + await expect( + ctx.configEditor.edit(piEntry, appendTo("opencode", SAVE_CANDIDATE)) + ).rejects.toThrow(S0_TODAY.saveRejectedUnknown); + expect(readFileSync(profile.patchPath, "utf8")).toBe(cleared); + }); + } +); + +describe( + "characterization · pi-ai applying first (tier B)", + { timeout: 30_000 }, + () => { + beforeEach(() => bootS0("pi-ai-first"), 30_000); + + it("[BASELINE] hasProviderApiKey answers the Zen plane with only the plane keys stored, and llm-pi-ai's settings.describe value lists both planes (F20, D18)", async () => { + await expectSignInReadsTheZenPlane(); + }); + + it("[INTENDED_CHANGE] the adapter decoration runs, and the base it repairs never reaches the wire", async () => { + const adapter = planeAdapter(); + expect(Object.hasOwn(adapter, "prepareCall")).toBe(true); + for (const [route, wireUrl] of [ + ["opencode", "https://opencode.ai/zen/v1/chat/completions"], + ["opencode-go", "https://opencode.ai/zen/go/v1/chat/completions"], + ] as const) { + // Measured: the Go plane's repaired base is ZEN's. pi-ai serves both + // planes from ONE adapter instance, which decoratePlaneAdapters wraps + // once per route, so the Go wrapper runs over the Zen one and keeps the + // base it already filled. + const decorated = "https://opencode.ai/inference/openai/v1"; + const prepared = await adapter.prepareCall(route, "deepseek-v4-pro"); + expect(prepared).toMatchObject({ + api: "openai-completions", + baseUrl: decorated, + }); + const { drained, requests } = await sentBy(route, "deepseek-v4-pro"); + expect(requests.map((request) => request.url)).toEqual([wireUrl]); + // Measured, not compared as literals: the URL the turn sent does not + // start with the base the decorated adapter prepared. + expect(requests[0]?.url.startsWith(String(prepared.baseUrl))).toBe( + false + ); + expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); + } + }); + + it("[KNOWN_DEFECT] the Go Gemini verdict depends on the apply order: decorated, gemini-4-argon-contributor validates, and no other facet of any row moves (decoratePlaneAdapters in src/models-discovery.ts)", async () => { + // The plan expected INVALID_CONFIG for Go Gemini; that holds in the + // published host's order (above) and not in this one, where the wrapped + // plane adapter delegates to opencode-go-google. todayRowsFor is the + // table the binary e2e picks when its probe infers this order. + const expected = todayRowsFor("pi-ai-first"); + expect( + ROW_KEYS.filter( + (key) => + JSON.stringify(expected[key]) !== JSON.stringify(S0_TODAY.rows[key]) + ) + ).toEqual(["opencode-go/gemini-4-argon-contributor"]); + for (const facet of ["info", "callConfig", "wire"] as const) { + const measured: Record = {}; + const want: Record = {}; + for (const key of ROW_KEYS) { + measured[key] = await measureFacet(facet, key); + want[key] = expectedFacet(facet, key, expected); + } + expect(measured, facet).toEqual(want); + } + }); + } +); + +describe( + "characterization · a gap-free plane (tier B)", + { timeout: 30_000 }, + () => { + it("[KNOWN_DEFECT] the unknown id itself is what the save check names when the plane stores no other gap", async () => { + // Plain S0 (no declared unknown id) minus Zen's stored gap id. + const providers = s0Providers(); + const zen = providers.opencode as { models: { id: string }[] }; + zen.models = zen.models.filter( + (model) => model.id !== "gpt-5.3-codex-spark" + ); + // The plugin's row mounts too, so its fetch patch wraps the recording + // fetch, as in every other case here, and not the suite's offline guard. + wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + tier = await bootTierB({ providers }); + const { ctx, piEntry, profile } = tier; + const before = readFileSync(profile.patchPath); + await expect( + ctx.configEditor.edit(piEntry, appendTo("opencode", SAVE_CANDIDATE)) + ).rejects.toThrow(S0_TODAY.saveRejectedUnknown); + expect(readFileSync(profile.patchPath)).toEqual(before); + }); + } +); diff --git a/test/host-credentials.test.ts b/test/host-credentials.test.ts new file mode 100644 index 00000000..8e0377a1 --- /dev/null +++ b/test/host-credentials.test.ts @@ -0,0 +1,144 @@ +/** + * pi-ai's credential paths, measured on the real package (tier A). + * + * D20 says the plane adapter resolves keys "by pi-ai's rule", and P-R1 builds + * its credential store from pi-ai's record mapping. Both are only as true as + * what pi-ai does today, so these cases drive a real turn through + * `LlmRuntime` and read the wire: which key the credentials service was asked + * for, whether the launch environment was consulted, and what authorization + * reached the request. Header values are recorded by LENGTH only, so no key + * text lands in an assertion or a log. + * + * Host references are into `deepseek-harness`'s `packages/llm/llm-pi-ai/src`. + * + * @module test/host-credentials.test + */ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + drainStream, + fakeCredentials, + fakeLaunchEnvironment, + mountTierA, + recordingFetch, + type TierA, +} from "./support/host.ts"; + +let tier: TierA | undefined; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; + vi.unstubAllGlobals(); +}); + +/** One completions model on Zen, which pi 1.1.0 serves at its legacy base. */ +const ZEN_COMPLETIONS = { models: [{ id: "deepseek-v4-pro" }] }; +const LEGACY_COMPLETIONS = "https://opencode.ai/zen/v1/chat/completions"; + +describe("host · pi-ai's credential paths (tier A)", () => { + it("maps an api-key record under llm-pi-ai/opencode onto the wire with no apiKeyEnv (auth.ts:74-84,141-148)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const credentials = fakeCredentials({ + records: { "llm-pi-ai/opencode": { key: "stored-k", kind: "api-key" } }, + }); + tier = await mountTierA({ + credentials, + providers: { opencode: ZEN_COMPLETIONS }, + }); + const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(wire.requests.map((request) => request.url)).toEqual([ + LEGACY_COMPLETIONS, + ]); + expect(wire.requests[0]?.headers.authorization).toBe( + "Bearer stored-k".length + ); + expect(credentials.calls).toContainEqual([ + "readRecord", + "llm-pi-ai/opencode", + ]); + expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); + }); + + it("F18 — a named ref the credentials service misses is MISSING_CREDENTIAL with no request, and listModels answers [] (index.ts:188-211; adapter.ts:295-312)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const credentials = fakeCredentials(); + const launchEnvironment = fakeLaunchEnvironment({ + OPENCODE_API_KEY: "env-k", + }); + tier = await mountTierA({ + credentials, + launchEnvironment, + providers: { + opencode: { ...ZEN_COMPLETIONS, apiKeyEnv: "OPENCODE_API_KEY" }, + }, + }); + expect(await tier.llm.listModels("opencode")).toEqual([]); + const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(drained.finish).toEqual({ + code: "MISSING_CREDENTIAL", + kind: "error", + }); + expect(wire.requests).toEqual([]); + expect(credentials.calls).toContainEqual(["resolve", "OPENCODE_API_KEY"]); + // With a credentials service mounted, the launch environment is never the + // fallback for a NAMED ref (index.ts:199-203). + expect(launchEnvironment.reads).not.toContain("OPENCODE_API_KEY"); + }); + + it("resolves a named ref through the credentials service and sends it (index.ts:199-205)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + tier = await mountTierA({ + credentials: fakeCredentials({ refs: { OPENCODE_API_KEY: "ref-k" } }), + providers: { + opencode: { ...ZEN_COMPLETIONS, apiKeyEnv: "OPENCODE_API_KEY" }, + }, + }); + const listed = await tier.llm.listModels("opencode"); + expect(listed.map((model) => model.id)).toEqual(["deepseek-v4-pro"]); + await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(wire.requests.map((request) => request.url)).toEqual([ + LEGACY_COMPLETIONS, + ]); + expect(wire.requests[0]?.headers.authorization).toBe("Bearer ref-k".length); + }); + + it("with no ref, pi's ambient auth asks the credentials service first, then the launch environment (auth.ts:203-215)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + // One log for both fakes, so the service calls and the environment read + // are ordered on the same clock. + const log: string[] = []; + const credentials = fakeCredentials({ sharedLog: log }); + const launchEnvironment = fakeLaunchEnvironment( + { OPENCODE_API_KEY: "env-k" }, + log + ); + tier = await mountTierA({ + credentials, + launchEnvironment, + providers: { opencode: ZEN_COMPLETIONS }, + }); + await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(wire.requests.map((request) => request.url)).toEqual([ + LEGACY_COMPLETIONS, + ]); + expect(wire.requests[0]?.headers.authorization).toBe("Bearer env-k".length); + // The stored record is read first, then the reference, then the + // environment: the credentials service is asked before the snapshot. + const at = (entry: string): number => { + const index = log.indexOf(entry); + expect(index, `${entry} in ${log.join(", ")}`).toBeGreaterThan(-1); + return index; + }; + expect(at("readRecord:llm-pi-ai/opencode")).toBeLessThan( + at("resolve:OPENCODE_API_KEY") + ); + expect(at("resolve:OPENCODE_API_KEY")).toBeLessThan( + at("env:OPENCODE_API_KEY") + ); + }); +}); diff --git a/test/host-editor-contracts.test.ts b/test/host-editor-contracts.test.ts new file mode 100644 index 00000000..1c9bd1e7 --- /dev/null +++ b/test/host-editor-contracts.test.ts @@ -0,0 +1,422 @@ +/** + * The config editor, the loader's volatile commit and the settings forms, + * pinned on the real host (tier B). + * + * The route-claim design keeps the user's file whole while pi-ai's resolved + * config loses the two planes (docs/improvement-plan.md, F12-F16, F20). That + * only works if the editor writes the raw value it was handed, if a commit can + * be told from the editor's validation run by identity, if a plane-only edit + * re-registers nothing, and if the settings page reads what these cases say + * it reads. Each case boots a fresh host and names the host file and line it + * pins; references are into `deepseek-harness`'s `packages/` and `vendor/`. + * + * These are host contracts, so the plugin's own row is left out (its hidden + * mount would add registry events of its own) except where the plugin's row is + * the subject (F16). + * + * @module test/host-editor-contracts.test + */ +import { readFileSync } from "node:fs"; + +import type { Context, Fiber } from "@deepseek-ai/cordis"; +import { readProfilePatches } from "@deepseek-ai/dsh-app-boot"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + bootTierB, + fakeCredentials, + planeKeysOnly, + providerWithApiKey, + recordingFetch, + s0Providers, + type TierB, + waterfallSpy, + withoutPlanes, +} from "./support/host.ts"; + +/** The listener the claim design installs, prepended on the root. */ +function stripPlanesForPiAi( + this: Fiber, + raw: unknown, + next: () => unknown +): unknown { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + const copy = withoutPlanes(raw); + next(); + return copy; +} + +const prependStrip = (root: Context): void => { + root.on("internal/config", stripPlanesForPiAi, { prepend: true }); +}; + +/** S0 without the ids pi 1.1.0 cannot place, so pi-ai's save check passes. */ +const knownProviders = (): Record> => { + const providers = s0Providers(); + const gaps = new Set([ + "gpt-5.3-codex-spark", + "gemini-4-argon-contributor", + "minimax-m3.1-flash", + ]); + for (const plane of ["opencode", "opencode-go"]) { + const entry = providers[plane] as { models: { id: string }[] }; + entry.models = entry.models.filter((model) => !gaps.has(model.id)); + } + return providers; +}; + +type ProviderDict = Record>; + +/** An edit of one provider entry, the way a settings surface derives one. */ +const editProvider = + (route: string, patch: Record) => + (current: Record): Record => { + const providers = current.providers as ProviderDict; + return { + ...current, + providers: { ...providers, [route]: { ...providers[route], ...patch } }, + }; + }; + +/** The last pi-ai row with a config in the profile, read through the host. */ +const savedPiRow = (tier: TierB): { providers: ProviderDict } => { + const row = readProfilePatches("dsh", tier.profile).findLast( + (patch) => patch.id === "llm-pi-ai" && patch.config !== undefined + ); + return row?.config as { providers: ProviderDict }; +}; + +const resolvedKeys = (fiber: Fiber | undefined): string[] => { + const config = fiber?.config as + | { providers: { get: () => Record } } + | undefined; + return Object.keys(config?.providers.get() ?? {}); +}; + +/** The provider keys one side of a settings descriptor holds. */ +const providerKeys = (side: unknown): string[] => + Object.keys((side as { providers?: object } | undefined)?.providers ?? {}); + +/** The refs `credentials.describe` was asked about from `from` on. */ +const describedRefs = (tier: TierB, from: number): string[] => + tier.credentials.calls + .slice(from) + .filter(([method]) => method === "describe") + .map(([, ref]) => ref); + +const routeIds = (tier: TierB): string[] => + tier.llm.listProviders().map((provider) => provider.id); + +let tier: TierB | undefined; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; + vi.unstubAllGlobals(); +}); + +describe( + "host · the config editor and the volatile commit (tier B)", + { + timeout: 30_000, + }, + () => { + it("writes the raw next to the profile YAML, not the resolved value (config-editor/src/index.ts:100-128,142)", async () => { + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: knownProviders(), + }); + expect(routeIds(tier)).toEqual(["deepseek"]); + await tier.ctx.configEditor.edit( + tier.piEntry, + editProvider("opencode", { displayName: "Zen (edited)" }) + ); + const saved = savedPiRow(tier); + expect(saved.providers.opencode?.displayName).toBe("Zen (edited)"); + expect(Object.keys(saved.providers).toSorted()).toEqual([ + "deepseek", + "opencode", + "opencode-go", + ]); + // The editor's own read of the file agrees. + const override = tier.ctx.configEditor + .configuration() + .find((row) => row.entry === tier?.piEntry)?.override as { + providers: ProviderDict; + }; + expect(override.providers.opencode?.displayName).toBe("Zen (edited)"); + // The fiber keeps the raw it was handed, and resolves the stripped copy. + expect(resolvedKeys(tier.piFiber)).toEqual(["deepseek"]); + const raw = ( + tier.piFiber as unknown as { _config: { providers: ProviderDict } } + )._config; + expect(raw.providers.opencode?.displayName).toBe("Zen (edited)"); + }); + + it("F14 — the editor's candidate is never fiber._config; _commitVolatile's run is (config-editor:103; loader entry.ts:149,171)", async () => { + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: knownProviders(), + }); + const booted = tier; + const spy = waterfallSpy(booted.ctx, () => booted.piEntry.fiber); + await booted.ctx.configEditor.edit( + booted.piEntry, + editProvider("opencode", { displayName: "Zen (edited)" }) + ); + expect(spy.records.map((record) => record.isCommit)).toEqual([ + false, + true, + ]); + const [, commit] = spy.records; + expect(commit?.raw).toBe(booted.piEntry.options.config); + expect(commit?.raw).toBe( + (booted.piFiber as unknown as { _config: unknown })._config + ); + expect(spy.records[0]?.raw).not.toBe(commit?.raw); + }); + + it("F15 — a plane-only edit emits no loader/volatile-update and restarts nothing; a sibling's edit emits one (loader entry.ts:177-191)", async () => { + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: knownProviders(), + }); + const volatile: unknown[] = []; + let adaptersUpdated = 0; + tier.ctx.on( + "loader/volatile-update", + (paths: unknown) => { + volatile.push(paths); + }, + { global: true } + ); + tier.ctx.on( + "llm/adapters-updated", + () => { + adaptersUpdated += 1; + }, + { global: true } + ); + const before = tier.piEntry.fiber; + await tier.ctx.configEditor.edit( + tier.piEntry, + editProvider("opencode", { displayName: "Zen (edited)" }) + ); + expect(volatile).toEqual([]); + expect(adaptersUpdated).toBe(0); + expect(tier.piEntry.fiber).toBe(before); + expect(tier.piFiber.state).toBe(2); + expect(routeIds(tier)).toEqual(["deepseek"]); + + // Control: an edit pi-ai DOES see commits one volatile update, and pi-ai + // re-registers in place — the route replace and the directory replace + // each announce (llm/src/index.ts:475,536) — without a restart. + await tier.ctx.configEditor.edit( + tier.piEntry, + editProvider("deepseek", { displayName: "DS (edited)" }) + ); + expect(volatile).toEqual([[["providers"]]]); + expect(adaptersUpdated).toBe(2); + expect(tier.piEntry.fiber).toBe(before); + expect(tier.llm.listProviders()).toEqual([ + { id: "deepseek", name: "DS (edited)" }, + ]); + }); + + it("the stripping listener does not shield pi-ai's own save check: with a gap id stored, a plane edit is refused and the file is untouched (llm-pi-ai/src/index.ts:177-186; config.ts:364-368)", async () => { + // Read for P-R1. An outermost listener's copy is what the fiber KEEPS, + // but pi-ai validates the value its inner chain returns — the editor's + // raw candidate — and against its stripped previous config every plane + // entry reads as changed, so the strict check runs on it. + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: s0Providers(), + }); + const booted = tier; + const spy = waterfallSpy(booted.ctx, () => booted.piEntry.fiber); + const before = readFileSync(booted.profile.patchPath); + await expect( + booted.ctx.configEditor.edit( + booted.piEntry, + editProvider("opencode", { displayName: "Zen (edited)" }) + ) + ).rejects.toThrow( + /provider "opencode" model "gpt-5\.3-codex-spark" needs an api/u + ); + expect(spy.records.map((record) => record.isCommit)).toEqual([false]); + expect(readFileSync(booted.profile.patchPath)).toEqual(before); + }); + } +); + +describe("host · settings.describe (tier B)", { timeout: 30_000 }, () => { + it("host contract, F20 — builds value from the resolved fiber config while user keeps the YAML's entries (settings/src/index.ts:319-322)", async () => { + tier = await bootTierB({ + plugin: false, + prepare: prependStrip, + providers: s0Providers(), + }); + const stripped = tier.ctx.settings + .describe() + .find((descriptor) => descriptor.ns === "llm-pi-ai"); + expect(providerKeys(stripped?.value)).toEqual(["deepseek"]); + expect(providerKeys(stripped?.user).toSorted()).toEqual([ + "deepseek", + "opencode", + "opencode-go", + ]); + expect(stripped?.applies).toBe("live"); + }); + + it("host contract, F20 control — with no listener, value carries every entry the YAML holds", async () => { + tier = await bootTierB({ plugin: false, providers: s0Providers() }); + const plain = tier.ctx.settings + .describe() + .find((descriptor) => descriptor.ns === "llm-pi-ai"); + expect(providerKeys(plain?.value).toSorted()).toEqual([ + "deepseek", + "opencode", + "opencode-go", + ]); + }); +}); + +describe( + "host · session-controller's hasProviderApiKey, restated (tier B, no plugin row)", + { timeout: 30_000 }, + () => { + // Host contracts, measured without the plugin, so nothing in src/ can move + // them. The with-plugin row P-R1 must keep green is the [BASELINE] case in + // test/host-characterization.test.ts. + it("host contract — with only the plane keys stored, the Zen plane answers it: its describe().value carries apiKeyEnv and the key is configured", async () => { + tier = await bootTierB({ + credentials: planeKeysOnly(), + plugin: false, + providers: s0Providers(), + }); + const from = tier.credentials.calls.length; + expect(await providerWithApiKey(tier)).toBe("opencode"); + expect(describedRefs(tier, from)).toContain("OPENCODE_API_KEY"); + }); + + it("host contract, control — with no key stored it answers false, and with only DEEPSEEK_API_KEY the sibling answers it", async () => { + tier = await bootTierB({ plugin: false, providers: s0Providers() }); + expect(await providerWithApiKey(tier)).toBeUndefined(); + await tier.dispose(); + tier = await bootTierB({ + credentials: fakeCredentials({ refs: { DEEPSEEK_API_KEY: "ds" } }), + plugin: false, + providers: s0Providers(), + }); + expect(await providerWithApiKey(tier)).toBe("deepseek"); + }); + + it("host contract, F20 — under a test-local strip it answers FALSE with the plane keys stored: the directory still lists the planes, but their describe().value is gone, so their keys are never asked about", async () => { + // Why P-R1 needs the settings.describe wrapper (D18): upstream, false + // here makes initializeDefaultModel replace the user's default with the + // account's first model. The strip is this file's own listener, not the + // plugin, so this case cannot flip; what P-R1 must keep green while its + // own strip is active is the with-plugin [BASELINE] in + // test/host-characterization.test.ts. + tier = await bootTierB({ + credentials: planeKeysOnly(), + plugin: false, + prepare: prependStrip, + providers: s0Providers(), + }); + for (const plane of ["opencode", "opencode-go"]) { + expect(tier.llm.listConfigurableProviders()).toContainEqual( + expect.objectContaining({ + provider: plane, + settingsNs: "llm-pi-ai", + settingsPath: ["providers", plane], + }) + ); + } + const from = tier.credentials.calls.length; + expect(await providerWithApiKey(tier)).toBeUndefined(); + expect(describedRefs(tier, from)).toEqual(["DEEPSEEK_API_KEY"]); + }); + } +); + +describe( + "host · !!js and the outermost listener (tier B)", + { timeout: 30_000 }, + () => { + it("F16 — the outermost listener sees an un-interpolated {__jsExpr} node; the fiber's config holds the evaluated value (loader/src/index.ts:104-113)", async () => { + // The expression evaluates to the NON-default value (`debug` defaults + // to false), so a value that fell back to the default cannot pass. + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const seen: unknown[] = []; + tier = await bootTierB({ + prepare: (root) => { + root.on( + "internal/config", + function recordPluginRaw( + this: Fiber, + raw: unknown, + next: () => unknown + ) { + if (this.runtime?.name === "dsh-opencode-patch") { + seen.push(raw); + } + return next(); + }, + { prepend: true } + ); + }, + profilePatchYaml: [ + "- id: dsh-opencode-patch", + " config:", + ' debug: !!js "1 < 2"', + "", + ].join("\n"), + }); + expect(seen.length).toBeGreaterThan(0); + expect((seen[0] as { debug: unknown }).debug).toEqual({ + __jsExpr: "1 < 2", + }); + const pluginConfig = tier.pluginEntry?.fiber?.config as + | { debug: unknown } + | undefined; + expect(pluginConfig?.debug).toBe(true); + }); + + it("F16 on pi-ai's row — a !!js node inside a provider profile boots evaluated, but pi-ai's own check refuses every later save of the row (llm-pi-ai/src/index.ts:177-180)", async () => { + // Read for P-R1: pi-ai's listener runs `Config(raw)` on the value its + // inner chain returns, which the loader has not interpolated yet, so a + // `!!js` node inside `providers` survives boot and blocks the editor. + tier = await bootTierB({ + plugin: false, + profilePatchYaml: [ + "- id: llm-pi-ai", + " config:", + " providers:", + " deepseek:", + " apiKeyEnv: DEEPSEEK_API_KEY", + ' displayName: !!js \'"D" + "S"\'', + "", + ].join("\n"), + }); + expect(tier.llm.listProviders()).toEqual([ + { id: "deepseek", name: "DS" }, + ]); + await expect( + tier.ctx.configEditor.edit( + tier.piEntry, + editProvider("deepseek", { baseURL: "http://127.0.0.1:9/v1" }) + ) + ).rejects.toThrow( + "$.providers.deepseek.displayName expected string but got [object Object]" + ); + }); + } +); diff --git a/test/host-harness.test.ts b/test/host-harness.test.ts new file mode 100644 index 00000000..4d122a8a --- /dev/null +++ b/test/host-harness.test.ts @@ -0,0 +1,239 @@ +/** + * The harness's own seams, each exercised once (`test/support/host.ts`). + * + * P-R1 builds on options no contract or characterization case passes: this + * plugin mounted in tier A, a profile row for it in tier B, and a custom + * responder on the recording fetch. A builder nobody has run is a builder + * whose behaviour a later phase would assume, so each is run here and what it + * does is recorded — including that tier A's plugin is a DEGRADED one. So are + * tier B's two failure paths: a first row whose config the schema rejects (in + * each apply order), and a key the contributor's shell exports. + * + * @module test/host-harness.test + */ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + adaptersOf, + type ApplyOrder, + bootTierB, + characterizedProviders, + drainStream, + fakeCredentials, + mountTierA, + PLANES, + recordingFetch, + S0_TODAY, + s0Providers, + type TierA, + type TierB, + waitForRoutes, +} from "./support/host.ts"; +import { INTERNAL_ROUTES, loadPiAiAdapterHost } from "./support/src.ts"; + +let tierA: TierA | undefined; +let tierB: TierB | undefined; + +afterEach(async () => { + try { + await tierA?.dispose(); + await tierB?.dispose(); + } finally { + tierA = undefined; + tierB = undefined; + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + } +}); + +/** cordis `FiberState`: the two a booted row ends in here. */ +const ACTIVE = 2; +const FAILED = 3; + +/** Poll until `predicate` holds, or fail after `deadlineMs`. */ +const until = async ( + predicate: () => boolean, + what: string, + deadlineMs = 10_000 +): Promise => { + const end = Date.now() + deadlineMs; + while (!predicate()) { + if (Date.now() > end) { + throw new Error(`never happened: ${what}`); + } + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + } +}; + +/** A two-chunk chat-completions stream that ends with `stop`. */ +const completionStream = (model: string): Response => { + const chunk = (delta: object, finish: string | null) => + `data: ${JSON.stringify({ + choices: [{ delta, finish_reason: finish, index: 0 }], + created: 1, + id: "harness", + model, + object: "chat.completion.chunk", + })}\n\n`; + return new Response( + `${chunk({ content: "hello", role: "assistant" }, null)}${chunk({}, "stop")}data: [DONE]\n\n`, + { headers: { "content-type": "text/event-stream" }, status: 200 } + ); +}; + +describe( + "host harness · its seams (tiers A and B)", + { timeout: 30_000 }, + () => { + it("tier A with the plugin: it applies and patches fetch, but WITHOUT a loader it cannot find pi-ai, so no hidden route mounts; disposal restores fetch", async () => { + const info = vi.spyOn(console, "info").mockImplementation(() => { + // Silenced: the case reads the calls, not the console. + }); + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + tierA = await mountTierA({ + plugin: {}, + providers: { opencode: { models: [{ id: "deepseek-v4-pro" }] } }, + }); + expect(tierA.pluginFiber?.state).toBe(2); + expect(globalThis.fetch).not.toBe(wire.fetch); + // The plugin finds pi-ai through the loader's entries + // (`piAiEntry` in src/responses-provider.ts), and tier A has no loader: the mount + // reports that and stands down. A P-R1 case that needs the real plugin + // beside pi-ai belongs in tier B. + await until( + () => + info.mock.calls.some((call) => + String(call[0]).includes("no loaded llm-pi-ai in this composition") + ), + "the mount's stand-down line" + ); + const registry = adaptersOf(tierA.llm); + for (const route of INTERNAL_ROUTES) { + expect(registry.has(route), route).toBe(false); + } + expect([...registry.keys()]).toEqual(["opencode"]); + // The Google bridge's absence is pinned by its CAUSE, not its outcome: + // this case declares no Google model, so an empty registry above would + // hold for that reason alone. The bridge builds its adapter from the + // record `loadPiAiAdapterHost` answers (src/responses-provider.ts), and + // registerGoogleBridge stands down without its `PiAiAdapter` + // (src/google-bridge.ts) — and without a loader there is no record. + await expect(loadPiAiAdapterHost(tierA.ctx)).resolves.toBeUndefined(); + await tierA.dispose(); + tierA = undefined; + expect(globalThis.fetch).toBe(wire.fetch); + }); + + it("tier B with pluginConfig: the profile row reaches the plugin's fiber config", async () => { + vi.stubGlobal("fetch", recordingFetch().fetch); + tierB = await bootTierB({ pluginConfig: { debug: true } }); + const config = tierB.pluginEntry?.fiber?.config as + | { debug?: unknown } + | undefined; + // `debug` defaults to false, so only the row can make it true. + expect(config?.debug).toBe(true); + }); + + it("recordingFetch with a responder: a turn is answered by it and still recorded, so a test can drive a stream to completion", async () => { + const wire = recordingFetch({ + respond: (request) => completionStream(request.model ?? ""), + }); + vi.stubGlobal("fetch", wire.fetch); + tierA = await mountTierA({ + credentials: fakeCredentials({ refs: { OPENCODE_API_KEY: "k" } }), + providers: { + opencode: { + apiKeyEnv: "OPENCODE_API_KEY", + models: [{ id: "deepseek-v4-pro" }], + }, + }, + }); + const drained = await drainStream( + tierA.llm, + "opencode", + "deepseek-v4-pro" + ); + expect(drained.error).toBeUndefined(); + expect(drained.finish).toEqual({ code: undefined, kind: "stop" }); + expect(drained.types).toContain("text-delta"); + expect(wire.requests.map((request) => request.model)).toEqual([ + "deepseek-v4-pro", + ]); + }); + it.each([ + [ + "plugin-first", + "an invalid plugin row", + { pluginConfig: { debug: "not-a-boolean" } }, + ], + [ + "pi-ai-first", + "an invalid pi-ai profile", + { providers: { deepseek: { displayName: 42 } } }, + ], + ] as const)( + "tier B, %s with %s: the first row FAILS without applying, the held row is still released, and the boot finishes promptly", + async (order, _what, rows) => { + // Before the fallback release, the held row waited for an `apply` that + // never came, and the boot hung until the hook timed out. + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + vi.stubGlobal("fetch", recordingFetch().fetch); + tierB = await bootTierB({ order: order as ApplyOrder, ...rows }); + const [failedEntry, heldEntry] = + order === "plugin-first" + ? [tierB.pluginEntry, tierB.piEntry] + : [tierB.piEntry, tierB.pluginEntry]; + expect(failedEntry?.fiber?.state).toBe(FAILED); + // The real error is readable, not a hook timeout. + await expect(failedEntry?.fiber?.await()).rejects.toThrow( + order === "plugin-first" ? /debug/u : /displayName/u + ); + expect(heldEntry?.fiber?.state).toBe(ACTIVE); + }, + 15_000 + ); + + it("tier B with OPENCODE_API_KEY exported: the boot blanks it, so the deployment measured is still the keyless one, and dispose puts it back", async () => { + vi.stubEnv("OPENCODE_API_KEY", "sk-ambient-not-a-key"); + vi.stubEnv("OPENCODE_GO_API_KEY", "sk-ambient-go-not-a-key"); + vi.stubGlobal("fetch", recordingFetch().fetch); + tierB = await bootTierB({ + credentials: fakeCredentials({ + refs: { + OPENCODE_API_KEY: "zen-test", + OPENCODE_GO_API_KEY: "go-test", + }, + }), + providers: characterizedProviders(), + }); + await waitForRoutes(tierB.llm, ["opencode-responses"]); + expect(process.env.OPENCODE_API_KEY).toBe(""); + expect(process.env.OPENCODE_GO_API_KEY).toBe(""); + // The plugin reads the environment for its keyless listing filter: with + // the shell's key left in place, the paid models would be listed here. + for (const plane of PLANES) { + expect(await tierB.llm.listModels(plane), plane).toEqual( + S0_TODAY.keylessListing[plane] + ); + } + await tierB.dispose(); + tierB = undefined; + expect(process.env.OPENCODE_API_KEY).toBe("sk-ambient-not-a-key"); + expect(process.env.OPENCODE_GO_API_KEY).toBe("sk-ambient-go-not-a-key"); + }); + + it("tier B leaves an unset key unset", async () => { + // oxlint-disable-next-line unicorn/no-useless-undefined -- `stubEnv` deletes the variable only when handed `undefined`, and restores it after. + vi.stubEnv("OPENCODE_API_KEY", undefined); + tierB = await bootTierB({ plugin: false, providers: s0Providers() }); + expect(process.env.OPENCODE_API_KEY).toBeUndefined(); + await tierB.dispose(); + tierB = undefined; + expect(process.env.OPENCODE_API_KEY).toBeUndefined(); + }); + } +); diff --git a/test/host-shapes.test.ts b/test/host-shapes.test.ts new file mode 100644 index 00000000..30ee2be1 --- /dev/null +++ b/test/host-shapes.test.ts @@ -0,0 +1,461 @@ +/** + * The host shapes and exports the plane provider (P-R1) is built from, pinned + * on the installed packages. + * + * The design reaches into pi-ai's private profile shape, its exported `Config` + * and `recordKeyFor`, the loader's exported `interpolate` and pi's public + * per-protocol exports (docs/improvement-plan.md, "Target architecture"). None + * of that is a promise the host made, so each piece is pinned here against the + * real package, with the host file and line it reads. A host bump that moves + * one turns this file red before any routing code depends on it. + * + * `test/mount-protocols.test.ts` already pins the four `api/*.lazy` factories + * against the catalog's SDKs; this file does not repeat it. + * + * @module test/host-shapes.test + */ +import { readFileSync } from "node:fs"; + +import { Context } from "@deepseek-ai/cordis"; +import { interpolate } from "@deepseek-ai/cordis-plugin-loader"; +import { resolveRetryPolicy } from "@deepseek-ai/dsh-llm"; +import * as LlmPiAi from "@deepseek-ai/dsh-llm-pi-ai"; +import z from "@deepseek-ai/schemastery"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + adaptersOf, + mountTierA, + s0Providers, + type TierA, +} from "./support/host.ts"; + +/** A schemastery node, as far as the role walk reads it. */ +interface SchemaNode { + readonly type?: string; + readonly meta?: { readonly role?: string }; + readonly dict?: Readonly>; + readonly inner?: SchemaNode; + readonly list?: readonly SchemaNode[]; +} + +/** + * The node types the role walk knows. A host bump that introduces another + * (a wrapper carrying `inner`, say) fails the pin by name instead of being + * walked blind. + */ +const KNOWN_NODE_TYPES = new Set([ + "any", + "array", + "boolean", + "const", + "dict", + "intersect", + "number", + "object", + "string", + "transform", + "union", +]); + +/** The path segment an `inner` node is walked under: none for a wrapper. */ +const INNER_SEGMENT: Readonly> = { + array: ["[]"], + dict: ["*"], +}; + +/** + * Every node under `node` that declares a role, as `[...path, role]`, and + * every node type visited. A dict's value is walked under `*`, an array's item + * under `[]`, and any other node's `inner` (a transform, or a wrapper a later + * schemastery adds) at its parent's path; union and intersect members share + * their parent's path too. + */ +const rolesUnder = ( + node: SchemaNode, + path: string[], + types = new Set() +): string[][] => { + const found: string[][] = []; + types.add(String(node.type)); + if (node.meta?.role !== undefined) { + found.push([...path, node.meta.role]); + } + if (node.type === "object") { + for (const [key, child] of Object.entries(node.dict ?? {})) { + found.push(...rolesUnder(child, [...path, key], types)); + } + } + if (node.inner !== undefined) { + found.push( + ...rolesUnder( + node.inner, + [...path, ...(INNER_SEGMENT[String(node.type)] ?? [])], + types + ) + ); + } + for (const member of node.list ?? []) { + found.push(...rolesUnder(member, path, types)); + } + return found; +}; + +/** A resolved provider profile as pi-ai's adapter holds it (private). */ +const profileOf = (tier: TierA, route: string): Record => { + const adapter = adaptersOf(tier.llm).get(route)?.adapter as unknown as { + config: { profiles: () => ReadonlyMap> }; + }; + const profile = adapter.config.profiles().get(route); + if (profile === undefined) { + throw new Error(`no resolved profile for ${route}`); + } + return profile; +}; + +/** A pi model, as far as these pins read one. */ +interface PiModel { + readonly api: string; + readonly baseUrl: string; +} + +/** Per protocol, how many models a pi provider carries and on which bases. */ +const byProtocol = ( + provider: unknown +): Record => { + const models = ( + provider as { getModels: () => readonly PiModel[] } + ).getModels(); + const out: Record = {}; + for (const model of models) { + const row = out[model.api] ?? { bases: [], count: 0 }; + out[model.api] = row; + row.count += 1; + if (!row.bases.includes(model.baseUrl)) { + row.bases.push(model.baseUrl); + } + } + return out; +}; + +let tier: TierA | undefined; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; +}); + +describe("host · pi-ai's Config (llm-pi-ai/src/config.ts)", () => { + it("resolves an empty profile to these defaults, and providers is volatile (config.ts:47-80,326-354)", () => { + const resolved = ( + LlmPiAi.Config({ providers: { x: {} } } as never) as unknown as { + providers: { get: () => Record }; + } + ).providers.get().x; + expect(resolved).toEqual({ + compat: { chatTemplateArgs: {}, chatTemplateKwargs: {} }, + defaultContextWindow: 262_144, + defaultInput: ["text"], + defaultMaxTokens: 32_768, + headers: {}, + maxRequestImageBytes: 20_971_520, + modelOverrides: {}, + models: [], + requestImageMaxBytes: 1_048_576, + requestImagePixelBudget: 4_194_304, + streamIdleTimeoutMs: 300_000, + thinkingBudgets: {}, + }); + const schema = LlmPiAi.Config as unknown as SchemaNode & { + dict: { providers: { meta: { volatile?: boolean } } }; + }; + expect(schema.dict.providers.meta.volatile).toBe(true); + }); + + it("resolves an absent retry policy to normal mode with five retries (dsh-llm retry-policy.ts:149)", () => { + expect(resolveRetryPolicy(undefined, "t")).toEqual({ + initialDelayMs: 500, + jitterRatio: 0.1, + maxDelayMs: 10_000, + maxRetries: 5, + mode: "normal", + retryableCodes: [ + "EMPTY_RESPONSE", + "RATE_LIMIT", + "SERVER", + "TIMEOUT", + "TRANSPORT", + ], + }); + }); + + it("carries no role('secret') under providers; apiKeyEnv is the only role, credential-ref (config.ts:327)", () => { + const schema = LlmPiAi.Config as unknown as SchemaNode; + const types = new Set(); + expect(rolesUnder(schema, [], types)).toEqual([ + ["providers", "*", "apiKeyEnv", "credential-ref"], + ]); + expect( + [...types].filter((type) => !KNOWN_NODE_TYPES.has(type)), + "a node type the walk does not know" + ).toEqual([]); + // The walk reads roles at every depth: a planted secret is reported. + const planted = z.object({ + providers: z.dict( + z.object({ + models: z.array(z.object({ token: z.string().role("secret") })), + }) + ), + }) as unknown as SchemaNode; + expect(rolesUnder(planted, [])).toEqual([ + ["providers", "*", "models", "[]", "token", "secret"], + ]); + // ...including one under a transform, whose `inner` is walked too. + const wrapped = z.object({ + providers: z.dict( + z.transform( + z.object({ token: z.string().role("secret") }), + (value: unknown) => value + ) + ), + }) as unknown as SchemaNode; + expect(rolesUnder(wrapped, [])).toEqual([ + ["providers", "*", "token", "secret"], + ]); + }); +}); + +describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,490-507)", () => { + it("declares exactly these fields in the published types", () => { + const text = readFileSync( + new URL( + "../node_modules/@deepseek-ai/dsh-llm-pi-ai/lib/types/config.d.ts", + import.meta.url + ), + "utf8" + ); + const start = text.indexOf("export interface ResolvedPiAiProviderProfile"); + expect(start).toBeGreaterThan(-1); + const block = text.slice(start, text.indexOf("\n}", start)); + // The heritage clause names what is inherited; the body what is resolved. + expect(block).toContain( + "extends Omit" + ); + const fields = [...block.matchAll(/^ {4}(\w+\??):/gmu)].map( + (match) => match[1] + ); + expect(fields).toEqual([ + "provider", + "displayName", + "apiKeyEnv?", + "streamIdleTimeoutMs", + "maxRequestImageBytes", + "requestImagePixelBudget", + "requestImageMaxBytes", + "retryPolicy", + "piProvider?", + "catalogError?", + "modelErrors", + "configuredMaxTokens", + ]); + }); + + it("resolves a profile with these keys at runtime, models destructured out", async () => { + tier = await mountTierA({ providers: s0Providers() }); + const profile = profileOf(tier, "opencode"); + expect(Object.keys(profile).toSorted()).toEqual([ + "apiKeyEnv", + "catalogError", + "compat", + "configuredMaxTokens", + "defaultContextWindow", + "defaultInput", + "defaultMaxTokens", + "displayName", + "headers", + "maxRequestImageBytes", + "modelErrors", + "modelOverrides", + "piProvider", + "provider", + "requestImageMaxBytes", + "requestImagePixelBudget", + "retryPolicy", + "streamIdleTimeoutMs", + "thinkingBudgets", + ]); + expect(profile.modelErrors).toBeInstanceOf(Map); + expect(profile.configuredMaxTokens).toBeInstanceOf(Map); + // A stored gap id is a deferred diagnostic, not a refusal (config.ts:487). + expect([...(profile.modelErrors as Map).keys()]).toEqual([ + "gpt-5.3-codex-spark", + ]); + expect(profile.catalogError).toMatch( + /model "gpt-5\.3-codex-spark" needs an api/u + ); + expect([ + ...( + profileOf(tier, "opencode-go").modelErrors as Map + ).keys(), + ]).toEqual(["gemini-4-argon-contributor", "minimax-m3.1-flash"]); + }); + + it("F1/F2 — a model-less plane resolves pi 1.1.0's whole catalog, per protocol, on the legacy bases", async () => { + tier = await mountTierA({ + providers: { opencode: {}, "opencode-go": {} }, + }); + const zen = profileOf(tier, "opencode"); + expect(zen.catalogError).toBeUndefined(); + expect(byProtocol(zen.piProvider)).toEqual({ + "anthropic-messages": { bases: ["https://opencode.ai/zen"], count: 18 }, + "google-generative-ai": { + bases: ["https://opencode.ai/zen/v1"], + count: 7, + }, + "openai-completions": { + bases: ["https://opencode.ai/zen/v1"], + count: 28, + }, + "openai-responses": { bases: ["https://opencode.ai/zen/v1"], count: 30 }, + }); + // F10: no Google model on Go. + expect(byProtocol(profileOf(tier, "opencode-go").piProvider)).toEqual({ + "anthropic-messages": { bases: ["https://opencode.ai/zen/go"], count: 5 }, + "openai-completions": { + bases: ["https://opencode.ai/zen/go/v1"], + count: 19, + }, + "openai-responses": { + bases: ["https://opencode.ai/zen/go/v1"], + count: 6, + }, + }); + }); +}); + +describe("host · pi-ai's credential key (llm-pi-ai/src/auth.ts:29-39; credentials/src/index.ts:69-78)", () => { + it("addresses a route's record as llm-pi-ai/, and refuses a key outside the grammar", () => { + expect(LlmPiAi.recordKeyFor("opencode")).toBe("llm-pi-ai/opencode"); + expect(LlmPiAi.recordKeyFor("opencode-go")).toBe("llm-pi-ai/opencode-go"); + expect(() => LlmPiAi.recordKeyFor("Not Valid")).toThrow(TypeError); + }); +}); + +describe("host · the loader's interpolate (vendor/loader/src/config/utils.ts:12-22; index.ts:19)", () => { + it("replaces {__jsExpr} nodes at any depth and leaves the rest alone", () => { + expect( + interpolate(new Context(), { + a: { __jsExpr: "1 + 1" }, + b: [{ __jsExpr: '"x"' }], + c: "c", + }) + ).toEqual({ a: 2, b: ["x"], c: "c" }); + }); +}); + +describe("host · pi 1.1.0's public exports (package.json exports ./providers/*, ./api/*, ./models)", () => { + it("opencodeProvider/opencodeGoProvider resolve keys through pi's ambient auth (providers/opencode.js:10-24; opencode-go.js:8-20)", async () => { + const { opencodeProvider } = + await import("@earendil-works/pi-ai/providers/opencode"); + const { opencodeGoProvider } = + await import("@earendil-works/pi-ai/providers/opencode-go"); + for (const provider of [opencodeProvider(), opencodeGoProvider()]) { + const auth = provider.auth as { + apiKey?: { + resolve: (input: { + ctx: { env: (name: string) => Promise }; + credential?: { key?: string }; + signal: AbortSignal; + }) => Promise< + { auth: { apiKey: string }; source: string } | undefined + >; + }; + }; + expect(typeof auth.apiKey?.resolve).toBe("function"); + const asked: string[] = []; + const resolved = await auth.apiKey?.resolve({ + ctx: { + env: async (name) => { + asked.push(name); + return name === "OPENCODE_API_KEY" ? "ambient" : undefined; + }, + }, + signal: new AbortController().signal, + }); + // Both planes read the one Zen variable: Go has no variable of its own. + expect(asked).toEqual(["OPENCODE_API_KEY"]); + expect(resolved).toMatchObject({ + auth: { apiKey: "ambient" }, + source: "OPENCODE_API_KEY", + }); + } + }); + + it("F10 — the Go provider has no Google API: a Google model streams pi's own 'no API implementation' error", async () => { + const { opencodeProvider } = + await import("@earendil-works/pi-ai/providers/opencode"); + const { opencodeGoProvider } = + await import("@earendil-works/pi-ai/providers/opencode-go"); + const apisOf = (provider: { getModels: () => readonly PiModel[] }) => + [...new Set(provider.getModels().map((model) => model.api))].toSorted(); + expect(apisOf(opencodeProvider())).toEqual([ + "anthropic-messages", + "google-generative-ai", + "openai-completions", + "openai-responses", + ]); + expect(apisOf(opencodeGoProvider())).toEqual([ + "anthropic-messages", + "openai-completions", + "openai-responses", + ]); + const gemini = opencodeProvider() + .getModels() + .find((model) => model.api === "google-generative-ai"); + expect(gemini).toBeDefined(); + const events: unknown[] = []; + for await (const event of opencodeGoProvider().stream( + { ...gemini, provider: "opencode-go" } as never, + { messages: [] } as never, + {} + )) { + events.push(event); + } + expect(JSON.stringify(events)).toContain( + 'Provider opencode-go has no API implementation for \\"google-generative-ai\\"' + ); + }); + + it("exports createProvider and withOpenCodeSessionHeader (models.js:527; providers/opencode-headers.js:15)", async () => { + const { createProvider } = await import("@earendil-works/pi-ai/models"); + expect(typeof createProvider).toBe("function"); + const { withOpenCodeSessionHeader } = + await import("@earendil-works/pi-ai/providers/opencode-headers"); + const wrapped = withOpenCodeSessionHeader({ + stream: (_model: unknown, _context: unknown, options: unknown) => options, + streamSimple: (_model: unknown, _context: unknown, options: unknown) => + options, + } as never) as { + streamSimple: ( + model: unknown, + context: unknown, + options: unknown + ) => unknown; + }; + expect(wrapped.streamSimple({}, {}, { sessionId: "ses_x" })).toEqual({ + headers: { "x-opencode-session": "ses_x" }, + sessionId: "ses_x", + }); + // A caller-set header is kept, whatever its case. + expect( + wrapped.streamSimple( + {}, + {}, + { headers: { "X-OpenCode-Session": "mine" }, sessionId: "ses_x" } + ) + ).toEqual({ + headers: { "X-OpenCode-Session": "mine" }, + sessionId: "ses_x", + }); + }); +}); diff --git a/test/host-waterfall.test.ts b/test/host-waterfall.test.ts new file mode 100644 index 00000000..8e1f45a6 --- /dev/null +++ b/test/host-waterfall.test.ts @@ -0,0 +1,285 @@ +/** + * The `internal/config` waterfall, pinned on the real host (tier A). + * + * The route-claim design (docs/improvement-plan.md, D13) rests on cordis and + * pi-ai behaving exactly as these cases say: pi-ai registers the keys of its + * RESOLVED config (F11), a listener another plugin prepends wraps pi-ai's + * fiber and runs outermost (F8), a commit is told from a validation by + * identity (F14), and the fiber keeps the outermost listener's return value + * without anyone mutating the input (F12). A pin that goes red here sends the + * design back to the plan. + * + * Host references are into `deepseek-harness` at the tag `host-versions.ts` + * names: `vendor/cordis/src/…` and `packages/llm/…`. + * + * @module test/host-waterfall.test + */ +import type { Context, Fiber } from "@deepseek-ai/cordis"; +import { LlmAdapter } from "@deepseek-ai/dsh-llm"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + deepFreeze, + mountTierA, + s0Providers, + type TierA, + waterfallSpy, + withoutPlanes, +} from "./support/host.ts"; + +/** The resolved provider keys pi-ai's fiber holds. */ +const resolvedKeys = (fiber: Fiber): string[] => + Object.keys( + ( + fiber.config as { providers: { get: () => Record } } + ).providers.get() + ).toSorted(); + +/** The raw config the fiber stores (`fiber._config`). */ +const rawOf = (fiber: Fiber): unknown => + (fiber as unknown as { _config: unknown })._config; + +/** The registered route ids, sorted. */ +const routeIds = (tier: TierA): string[] => + tier.llm + .listProviders() + .map((provider) => provider.id) + .toSorted(); + +/** An adapter that registers and never streams: the claim's stand-in. */ +class StubAdapter extends LlmAdapter { + // oxlint-disable-next-line require-yield -- never streams; the registry is the subject. + async *stream(): AsyncIterable { + throw new Error("never streams"); + } +} + +/** + * The listener the claim design installs: for pi-ai's fiber only, the inner + * chain runs and the fiber keeps a fresh copy without the planes. + */ +function stripPlanesForPiAi( + this: Fiber, + raw: unknown, + next: () => unknown +): unknown { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + const copy = withoutPlanes(raw); + next(); + return copy; +} + +let tier: TierA | undefined; + +afterEach(async () => { + await tier?.dispose(); + tier = undefined; +}); + +describe("host · the internal/config waterfall (tier A)", () => { + it("F11 — pi-ai registers exactly the keys of its resolved config (llm-pi-ai/src/index.ts:168-176,299,307)", async () => { + tier = await mountTierA({ providers: s0Providers() }); + expect(routeIds(tier)).toEqual(resolvedKeys(tier.piFiber)); + expect(routeIds(tier)).toEqual(["deepseek", "opencode", "opencode-go"]); + }); + + it("F11 — the RESOLVED keys, not the raw ones: with a listener stripping the planes, the routes follow the resolved config while the raw still holds all three", async () => { + tier = await mountTierA({ + before: (ctx) => { + ctx.on("internal/config", stripPlanesForPiAi, { prepend: true }); + }, + providers: s0Providers(), + }); + expect(routeIds(tier)).toEqual(resolvedKeys(tier.piFiber)); + expect(routeIds(tier)).toEqual(["deepseek"]); + expect( + Object.keys( + (rawOf(tier.piFiber) as { providers: Record }) + .providers + ).toSorted() + ).toEqual(["deepseek", "opencode", "opencode-go"]); + }); + + it("F8/F12 — a listener another plugin prepends fires for pi-ai's fiber with this === fiber, outermost (cordis events.ts:234-243,254-260)", async () => { + tier = await mountTierA({ providers: s0Providers() }); + const order: string[] = []; + const seen: { name: string; self: unknown; raw: unknown }[] = []; + // A sibling plugin, mounted AFTER pi-ai: its plain listener lands after + // pi-ai's own (inner), its prepended one before everything (outer). + await tier.ctx.plugin({ + apply(c: Context) { + c.on( + "internal/config", + function listenerA(this: Fiber, raw: unknown, next: () => unknown) { + if (this.runtime?.name === "llm-pi-ai") { + order.push("A"); + seen.push({ name: "A", raw, self: this }); + } + return next(); + } + ); + c.on( + "internal/config", + function listenerB(this: Fiber, raw: unknown, next: () => unknown) { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + order.push("B:enter"); + seen.push({ name: "B", raw, self: this }); + const value = next(); + order.push("B:exit"); + return value; + }, + { prepend: true } + ); + }, + name: "claimer", + }); + const input = { providers: s0Providers() }; + tier.piFiber.update(input); + await tier.piFiber.await(); + // Two runs: `update` resolves while ACTIVE (fiber.ts:747), then the + // restart re-resolves the stored raw (fiber.ts:655). B brackets A both + // times, so B is outermost and A runs inside the chain B started. + expect(order).toEqual(["B:enter", "A", "B:exit", "B:enter", "A", "B:exit"]); + for (const record of seen) { + expect(record.self).toBe(tier.piFiber); + } + expect(seen[0]?.raw).toBe(input); + expect(tier.piFiber.state).toBe(2); + }); + + it("F8 — pi-ai's own listener validates the value its inner chain returns, not what an outer listener will return (llm-pi-ai/src/index.ts:177-186; config.ts:364-368)", async () => { + // Read for P-R1: an outer listener that strips the planes and returns a + // copy does NOT shield pi-ai's save check, which runs on `next()`'s value. + // A listener INSIDE pi-ai's (registered after it, unprepended) is what + // that check sees. + tier = await mountTierA({ providers: s0Providers() }); + const order: string[] = []; + await tier.ctx.plugin({ + apply(c: Context) { + c.on( + "internal/config", + function addUnknownId( + this: Fiber, + raw: unknown, + next: () => unknown + ) { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + order.push("A"); + next(); + // A changed plane holding an id pi's catalog does not describe. + const candidate = structuredClone(raw) as { + providers: Record; + }; + candidate.providers.opencode?.models?.push({ + id: "not-a-real-model", + }); + return candidate; + } + ); + c.on( + "internal/config", + function bracket(this: Fiber, _raw: unknown, next: () => unknown) { + if (this.runtime?.name !== "llm-pi-ai") { + return next(); + } + order.push("B:enter"); + try { + return next(); + } catch (error) { + order.push("B:pi-ai threw"); + throw error; + } + }, + { prepend: true } + ); + }, + name: "claimer", + }); + expect(() => { + tier?.piFiber.update({ providers: s0Providers() }); + // The strict check covers the whole changed plane, so it names the + // first id it cannot place: S0's stored gap model, ahead of the one A + // added. An unchanged plane is not checked at all (the case above). + }).toThrow( + /provider "opencode" model "gpt-5\.3-codex-spark" needs an api/u + ); + expect(order).toEqual(["B:enter", "A", "B:pi-ai threw"]); + }); + + it("F14 — raw === fiber._config on _reload and on update (cordis fiber.ts:641-644,655,738,747)", async () => { + tier = await mountTierA({ providers: s0Providers() }); + const { piFiber } = tier; + const spy = waterfallSpy(tier.ctx, () => piFiber); + await piFiber.restart(); + expect(spy.records).toHaveLength(1); + expect(spy.records[0]?.isCommit).toBe(true); + expect(spy.records[0]?.raw).toBe(rawOf(piFiber)); + + spy.records.length = 0; + const next = structuredClone(rawOf(piFiber)); + piFiber.update(next); + await piFiber.await(); + expect(spy.records.map((record) => record.isCommit)).toEqual([true, true]); + for (const record of spy.records) { + expect(record.raw).toBe(next); + } + expect(rawOf(piFiber)).toBe(next); + }); + + it("F12 — a deep-frozen input is never mutated, and the fiber keeps the outermost listener's return value (cordis fiber.ts:642,655; utils.ts:117-125)", async () => { + const frozen = deepFreeze({ providers: s0Providers() }); + tier = await mountTierA({ + before: (ctx) => { + ctx.on("internal/config", stripPlanesForPiAi, { prepend: true }); + }, + piConfig: frozen, + providers: {}, + }); + const { piFiber } = tier; + expect(rawOf(piFiber)).toBe(frozen); + expect( + Object.keys((rawOf(piFiber) as typeof frozen).providers).toSorted() + ).toEqual(["deepseek", "opencode", "opencode-go"]); + expect(resolvedKeys(piFiber)).toEqual(["deepseek"]); + expect(routeIds(tier)).toEqual(["deepseek"]); + }); + + it("F13 (tier A half) — with the planes stripped, registerAdapter(planes) passes the duplicate check and the directory still lists both under llm-pi-ai (llm/src/index.ts:401-445; llm-pi-ai/src/index.ts:128-150)", async () => { + tier = await mountTierA({ + before: (ctx) => { + ctx.on("internal/config", stripPlanesForPiAi, { prepend: true }); + }, + providers: s0Providers(), + }); + expect(routeIds(tier)).toEqual(["deepseek"]); + const handle = tier.llm.registerAdapter( + ["opencode", "opencode-go"], + new StubAdapter() + ); + try { + expect(routeIds(tier)).toEqual(["deepseek", "opencode", "opencode-go"]); + const directory = tier.llm.listConfigurableProviders(); + for (const id of ["opencode", "opencode-go"]) { + expect(directory.find((entry) => entry.provider === id)).toMatchObject({ + settingsNs: "llm-pi-ai", + settingsPath: ["providers", id], + }); + } + } finally { + handle(); + } + }); + + it("F13 control — while pi-ai holds the planes, registering them is DUPLICATE_ADAPTER (llm/src/index.ts:443)", async () => { + tier = await mountTierA({ providers: s0Providers() }); + expect(() => + tier?.llm.registerAdapter(["opencode"], new StubAdapter()) + ).toThrow(expect.objectContaining({ code: "DUPLICATE_ADAPTER" })); + }); +}); diff --git a/test/keyless-matrix.test.ts b/test/keyless-matrix.test.ts new file mode 100644 index 00000000..f616e059 --- /dev/null +++ b/test/keyless-matrix.test.ts @@ -0,0 +1,352 @@ +/** + * The keyless probe matrix's cell table and classifier, offline. + * + * The live half (`test/e2e/keyless-matrix.e2e.ts`) cannot run in a sandbox + * without the vendor's hosts, so everything it decides is decided here: which + * sixteen URLs are probed, with which bodies, and how an answer is read. The + * URLs are written out from the vendor's endpoint layout, not derived from the + * module, and the bundled-catalog case checks invariants rather than model + * names, so the nightly catalog refresh cannot turn it red. + * + * @module test/keyless-matrix.test + */ +import { describe, expect, it } from "vitest"; + +import { PLANES, S0_TODAY } from "./support/host.ts"; +import { + cellKey, + classifyKeylessProbe, + isFreeSpec, + isTransientProbeAnswer, + keylessMatrix, + MATRIX_PROTOCOLS, + type MatrixCell, + matrixUrl, + NEXT_DISPATCH, + TODAY_DISPATCH, +} from "./support/keyless-matrix.ts"; +import { + type CatalogModelSpec, + OPENCODE_GO_CATALOG, + OPENCODE_ZEN_CATALOG, + protocolFor, +} from "./support/src.ts"; + +const spec = ( + id: string, + provider_npm?: string, + free = false +): CatalogModelSpec => ({ + context_window: 1000, + id, + input_modalities: ["text"], + max_output_tokens: 100, + name: id, + ...(free ? { is_free: true } : {}), + ...(provider_npm === undefined ? {} : { provider_npm }), +}); + +/** + * One paid model per protocol on each plane, each behind a free one (flagged, + * or only named `free`) and Zen's Responses model behind a retired one too. + * A free model answers a keyless request with a real completion, so the cells + * must never pick one. + */ +const ZEN = [ + spec("zen-beam", undefined, true), + spec("zen-chat-free"), + spec("zen-chat"), + spec("zen-resp-free", "@ai-sdk/openai", true), + spec("zen-old-resp", "@ai-sdk/openai"), + spec("zen-resp", "@ai-sdk/openai"), + spec("zen-claude-free", "@ai-sdk/anthropic", true), + spec("zen-claude", "@ai-sdk/anthropic"), + spec("zen-gemini-free", "@ai-sdk/google", true), + spec("zen-gemini", "@ai-sdk/google"), + spec("zen-mistral", "@ai-sdk/mistral"), +]; +const GO = [ + spec("go-beam", undefined, true), + spec("go-chat"), + spec("go-resp-free", "@ai-sdk/openai"), + spec("go-resp", "@ai-sdk/openai"), + spec("go-claude", "@ai-sdk/anthropic"), + spec("go-gemini-free", "@ai-sdk/google", true), + spec("go-gemini", "@ai-sdk/google"), +]; + +const key = (cell: MatrixCell): string => cellKey(cell); + +const CREDENTIAL_HEADERS = new Set([ + "authorization", + "x-api-key", + "x-goog-api-key", +]); + +describe("keyless matrix · the cells", () => { + it("is sixteen cells with exactly these URLs and models", () => { + const cells = keylessMatrix({ + go: GO, + isDeprecated: (plane, id) => + plane === "opencode" && id === "zen-old-resp", + zen: ZEN, + }); + expect(cells).toHaveLength(16); + expect( + Object.fromEntries( + cells.map((cell) => [key(cell), [cell.url, cell.model]]) + ) + ).toEqual({ + "opencode openai-completions legacy": [ + "https://opencode.ai/zen/v1/chat/completions", + "zen-chat", + ], + "opencode openai-completions modern": [ + "https://opencode.ai/inference/openai/v1/chat/completions", + "zen-chat", + ], + "opencode openai-responses legacy": [ + "https://opencode.ai/zen/v1/responses", + "zen-resp", + ], + "opencode openai-responses modern": [ + "https://opencode.ai/inference/openai/v1/responses", + "zen-resp", + ], + "opencode anthropic-messages legacy": [ + "https://opencode.ai/zen/v1/messages", + "zen-claude", + ], + "opencode anthropic-messages modern": [ + "https://opencode.ai/inference/anthropic/v1/messages", + "zen-claude", + ], + "opencode google-generative-ai legacy": [ + "https://opencode.ai/zen/v1/models/zen-gemini:streamGenerateContent?alt=sse", + "zen-gemini", + ], + "opencode google-generative-ai modern": [ + "https://opencode.ai/inference/google/v1beta/models/zen-gemini:streamGenerateContent?alt=sse", + "zen-gemini", + ], + "opencode-go openai-completions legacy": [ + "https://opencode.ai/zen/go/v1/chat/completions", + "go-chat", + ], + "opencode-go openai-completions modern": [ + "https://opencode.ai/inference/go/openai/v1/chat/completions", + "go-chat", + ], + "opencode-go openai-responses legacy": [ + "https://opencode.ai/zen/go/v1/responses", + "go-resp", + ], + "opencode-go openai-responses modern": [ + "https://opencode.ai/inference/go/openai/v1/responses", + "go-resp", + ], + "opencode-go anthropic-messages legacy": [ + "https://opencode.ai/zen/go/v1/messages", + "go-claude", + ], + "opencode-go anthropic-messages modern": [ + "https://opencode.ai/inference/go/anthropic/v1/messages", + "go-claude", + ], + "opencode-go google-generative-ai legacy": [ + "https://opencode.ai/zen/go/v1/models/go-gemini:streamGenerateContent", + "go-gemini", + ], + "opencode-go google-generative-ai modern": [ + "https://opencode.ai/inference/go/google/v1beta/models/go-gemini:streamGenerateContent?alt=sse", + "go-gemini", + ], + }); + }); + + it("carries the model in the body for the three body-addressed protocols, and the smallest output budget", () => { + const cells = keylessMatrix({ go: GO, zen: ZEN }); + const zen = (protocol: string, family: string) => + cells.find( + (cell) => + cell.plane === "opencode" && + cell.protocol === protocol && + cell.family === family + ); + expect(zen("openai-completions", "modern")?.body).toEqual({ + max_tokens: 1, + messages: [{ content: "hi", role: "user" }], + model: "zen-chat", + }); + expect(zen("openai-responses", "legacy")?.body).toEqual({ + input: "hi", + max_output_tokens: 1, + model: "zen-old-resp", + }); + expect(zen("anthropic-messages", "modern")?.body).toEqual({ + max_tokens: 1, + messages: [{ content: "hi", role: "user" }], + model: "zen-claude", + }); + expect(zen("google-generative-ai", "modern")?.body).toEqual({ + contents: [{ parts: [{ text: "hi" }], role: "user" }], + generationConfig: { maxOutputTokens: 1 }, + }); + expect(zen("anthropic-messages", "legacy")?.headers).toEqual({ + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }); + expect(zen("openai-completions", "legacy")?.headers).toEqual({ + "content-type": "application/json", + }); + }); + + it("never picks a free model, flagged or only named so", () => { + const cells = keylessMatrix({ go: GO, zen: ZEN }); + const all = [...ZEN, ...GO]; + for (const cell of cells) { + const picked = all.find((entry) => entry.id === cell.model); + expect(picked, key(cell)).toBeDefined(); + expect(picked?.is_free, key(cell)).not.toBe(true); + expect(cell.model, key(cell)).not.toMatch(/free/iu); + } + // A plane whose only model for a protocol is free has no cell model. + const onlyFree = keylessMatrix({ + go: GO, + zen: [spec("zen-beam", undefined, true)], + }); + const zenCells = onlyFree.filter((cell) => cell.plane === "opencode"); + expect(zenCells).toHaveLength(8); + expect(zenCells.filter((cell) => cell.model !== undefined)).toEqual([]); + expect(isFreeSpec(spec("paid"))).toBe(false); + expect(isFreeSpec(spec("x", undefined, true))).toBe(true); + expect(isFreeSpec(spec("big-FREE-model"))).toBe(true); + }); + + it("reports a protocol a plane has no model for as a model-less, body-less cell", () => { + const cells = keylessMatrix({ + go: GO.filter((entry) => entry.provider_npm !== "@ai-sdk/google"), + zen: ZEN, + }); + const missing = cells.filter((cell) => cell.model === undefined); + expect(missing.map(key)).toEqual([ + "opencode-go google-generative-ai legacy", + "opencode-go google-generative-ai modern", + ]); + for (const cell of missing) { + expect(cell.body).toBeUndefined(); + } + }); + + it("on the bundled catalogs: sixteen cells, each the first PAID model of its protocol, and none can carry a key", () => { + const cells = keylessMatrix({ + go: OPENCODE_GO_CATALOG, + zen: OPENCODE_ZEN_CATALOG, + }); + expect(cells).toHaveLength(16); + for (const cell of cells) { + const catalog = + cell.plane === "opencode" ? OPENCODE_ZEN_CATALOG : OPENCODE_GO_CATALOG; + const first = catalog.find( + (entry) => + protocolFor(entry) === cell.protocol && + entry.is_free !== true && + !/free/iu.test(entry.id) + ); + expect(cell.model, key(cell)).toBe(first?.id); + expect(cell.model, key(cell)).toBeDefined(); + expect( + catalog.find((entry) => entry.id === cell.model)?.is_free, + key(cell) + ).not.toBe(true); + if (cell.protocol === "google-generative-ai") { + expect(cell.url).toContain( + `/models/${cell.model}:streamGenerateContent` + ); + } else { + expect(cell.body?.model).toBe(cell.model); + } + expect(cell.method).toBe("POST"); + expect( + Object.keys(cell.headers).filter((header) => + CREDENTIAL_HEADERS.has(header.toLowerCase()) + ) + ).toEqual([]); + expect(new URL(cell.url).hostname).toBe("opencode.ai"); + } + }); +}); + +describe("keyless matrix · the cells the live run asserts", () => { + it("TODAY_DISPATCH is exactly the cells S0_TODAY's measured turns were sent to", () => { + // Each URL a characterized turn sent is matched to the one cell whose PATH + // it is (the query aside: Messages adds `?beta=true`), with the row's own + // model in the Google path. A row that sent nothing adds nothing. + const reached = new Set(); + for (const [rowKey, row] of Object.entries(S0_TODAY.rows)) { + const slash = rowKey.indexOf("/"); + const plane = rowKey.slice(0, slash) as (typeof PLANES)[number]; + const model = rowKey.slice(slash + 1); + expect(PLANES, rowKey).toContain(plane); + for (const sent of row.sent) { + const matches = MATRIX_PROTOCOLS.flatMap((protocol) => + (["legacy", "modern"] as const) + .filter( + (family) => + new URL(matrixUrl(plane, protocol, family, model)).pathname === + new URL(sent).pathname + ) + .map((family) => cellKey({ family, plane, protocol })) + ); + expect(matches, `${rowKey} ${sent}`).toHaveLength(1); + reached.add(matches[0] ?? ""); + } + } + expect([...reached].toSorted()).toEqual([...TODAY_DISPATCH].toSorted()); + // Completions reach the legacy family today: the modern completions cells + // are P-R1's target, asserted live as a forward guard, never as today's. + expect([...TODAY_DISPATCH]).toContain("opencode openai-completions legacy"); + expect( + [...NEXT_DISPATCH].filter((cell) => TODAY_DISPATCH.has(cell)) + ).toEqual([]); + expect([...NEXT_DISPATCH].toSorted()).toEqual([ + "opencode openai-completions modern", + "opencode-go openai-completions modern", + ]); + const all = new Set(keylessMatrix({ go: GO, zen: ZEN }).map(key)); + for (const cell of [...TODAY_DISPATCH, ...NEXT_DISPATCH]) { + expect(all.has(cell), cell).toBe(true); + } + }); +}); + +describe("keyless matrix · reading an answer", () => { + it.each([ + [404, "", "path-gone"], + [401, '{"error":{"type":"AuthError"}}', "path-exists"], + [403, "", "path-exists"], + [410, "", "model-gone"], + [400, '{"error":{"message":"Model is unavailable"}}', "model-gone"], + [400, "model is UNAVAILABLE for this plan", "model-gone"], + [400, "Invalid API key", "path-exists"], + [200, "{}", "other"], + [500, "", "other"], + ] as const)("%i %s → %s", (status, bodyText, verdict) => { + expect(classifyKeylessProbe({ bodyText, status })).toBe(verdict); + }); +}); + +describe("keyless matrix · a transient answer is retried once", () => { + it.each([ + [undefined, true], + [429, true], + [500, true], + [503, true], + [401, false], + [404, false], + [400, false], + [200, false], + ] as const)("%s → %s", (status, retry) => { + expect(isTransientProbeAnswer(status)).toBe(retry); + }); +}); diff --git a/test/support/host.ts b/test/support/host.ts new file mode 100644 index 00000000..9f486934 --- /dev/null +++ b/test/support/host.ts @@ -0,0 +1,1416 @@ +/** + * The real host, in-process: the harness the routing work is judged against. + * + * Two tiers, both built from the host packages this repository installs at the + * versions `host-versions.ts` names (pi 1.1.0 patched, the `@deepseek-ai` + * cluster at the published host's tag): + * + * - **Tier A** ({@link mountTierA}): a bare cordis `Context`, `LlmRuntime`, and + * `dsh-llm-pi-ai` mounted as a plugin with a `{ providers }` config, so its + * config is the schemastery instance and its fiber has `update()` and + * `restart()`. No loader, no profile, no files. It is the tier for the + * `internal/config` waterfall and pi-ai's credential paths: anything that + * needs a fiber and the registry, and nothing that reads a profile. Mirrors + * the harness's own `llm/llm-pi-ai/tests/egress.spec.ts:33-47`. + * - **Tier B** ({@link bootTierB}): dsh-app-boot's `boot()` with a temporary + * HOME, a test bundle and a profile patch, exactly as the harness's + * `experimental/speech-to-text/tests/configuration.spec.ts:12-33` boots one + * (`initProfile`, `readProfilePatches`, `root.provide("profileContext")`). + * It is the tier for the config editor, `settings.describe`, the loader's + * volatile commit, and the characterization of the plugin as it ships: + * everything that reads the loader's entries or writes the profile YAML. + * + * **Tier B's rows carry PACKAGE names, not `cordis:` builtins.** The five host + * rows (`llm`, `llm-pi-ai`, `config-editor`, `settings`, and this plugin) are + * named `@deepseek-ai/dsh-llm` … `dsh-opencode-patch`, and a stub module loader + * on `root.loader.internal` resolves those names to the statically imported + * namespaces — the harness's own pattern for a composition test + * (`llm/llm-pi-ai/tests/loader-composition.spec.ts:62-79`). A builtin row is + * named `cordis:` (the loader's `vendor/loader/src/config/tree.ts:112-115`), + * and the plugin finds pi-ai's entry by `options.name === + * "@deepseek-ai/dsh-llm-pi-ai"` (`piAiEntry` in `src/responses-provider.ts`), + * so a builtin-named row would silently drop today's hidden-route mount and the + * Google bridge from everything this harness characterizes. + * + * **Tier B fixes the apply order.** The loader starts rows in parallel, and the + * stub loader resolves every import at once, so which of pi-ai and the plugin + * applies first would otherwise be an accident of row order. It matters: the + * plugin's adapter decoration wraps only what is registered when it applies. + * The default is `plugin-first`, the order measured on the published binary + * ({@link ApplyOrder}); `pi-ai-first` is the other one. The held row is + * released when the first row's `apply` returns OR its fiber settles in any + * state, so a first row whose config fails to resolve leaves the boot to + * finish with that fiber FAILED instead of hanging it. + * + * **Nothing here reads the process environment.** Both tiers provide a fake + * `credentials` service ({@link fakeCredentials}) and a launch-environment + * snapshot ({@link fakeLaunchEnvironment}) on the root, which is where pi-ai + * looks first (`launchEnvironmentOf` reads `ctx.get("launchEnvironment")` + * before it falls back to `process.env`). The plugin itself still reads + * `process.env.OPENCODE_API_KEY`/`OPENCODE_GO_API_KEY` for its keyless listing + * filter (`hasPlaneCredential` in `src/lifecycle.ts`), so tier B blanks either + * one a contributor's shell exports for the life of the boot and restores it + * on dispose: the suite measures the keyless deployment whatever the shell + * holds. + * + * **Offline.** {@link recordingFetch} answers every `opencode.ai` request with a + * 401 and refuses every other host. Install it with `vi.stubGlobal("fetch", …)` + * BEFORE mounting the plugin — its fetch patch captures `globalThis.fetch` at + * effect time (`installFetchPatch` in `src/lifecycle.ts`) — and restore it with + * `vi.unstubAllGlobals()` AFTER disposal, or the patch's own restore check + * leaves the stub in place. + * + * Every builder returns a `dispose` that the caller runs in a `finally` or an + * `afterEach`. + * + * @module test/support/host + */ +import { + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { Context, type Fiber } from "@deepseek-ai/cordis"; +import type { Entry, ModuleLoaderV2 } from "@deepseek-ai/cordis-plugin-loader"; +import { + boot, + initProfile, + type ProfileContext, + readProfilePatches, +} from "@deepseek-ai/dsh-app-boot"; +import { ConfigEditor } from "@deepseek-ai/dsh-config-editor"; +import { LlmRuntime } from "@deepseek-ai/dsh-llm"; +import * as LlmPiAi from "@deepseek-ai/dsh-llm-pi-ai"; +import Settings from "@deepseek-ai/dsh-settings"; + +import { apply, Config, inject, name } from "./src.ts"; + +/** The package name the plugin looks pi-ai up by (`PI_AI_PACKAGE` in `src/responses-provider.ts`). */ +export const PI_AI_PACKAGE = "@deepseek-ai/dsh-llm-pi-ai"; + +/** The `llm` service as the tests read it, private registry included. */ +export type HostLlm = Context["llm"]; + +/** One registry row, as `LlmRuntime`'s private `adapters` map holds it. */ +export interface RegistryRow { + readonly adapter: { + readonly constructor: { readonly name: string }; + readonly [key: string]: unknown; + }; + readonly provider: { readonly id: string; readonly name: string }; + readonly retryPolicy: unknown; +} + +/** + * The registry map. Private in the type, readable at runtime, and the only way + * to see a route `listProviders()` hides. + */ +export const adaptersOf = (llm: HostLlm): Map => + (llm as unknown as { adapters: Map }).adapters; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +/** One call the fake credentials service answered, values never included. */ +export type CredentialCall = readonly [method: string, key: string]; + +/** What {@link fakeCredentials} is seeded with. */ +export interface CredentialSeed { + /** + * A log shared with {@link fakeLaunchEnvironment}: each call appends + * `:` here too, so a test can order a credentials call against + * an environment read on one clock. + */ + readonly sharedLog?: string[]; + /** Reference values by name, e.g. `{ OPENCODE_API_KEY: "k" }`. */ + readonly refs?: Readonly>; + /** Records by joined key, e.g. `{ "llm-pi-ai/opencode": { kind: "api-key", key: "k" } }`. */ + readonly records?: Readonly>; +} + +/** The fake, with the log a test reads. */ +export interface FakeCredentials { + readonly calls: CredentialCall[]; + readonly service: Record unknown>; +} + +/** + * A `credentials` service over two in-memory maps: references (the + * `OPENCODE_API_KEY` form) and records (the `llm-pi-ai/opencode` form). It + * implements the surface pi-ai and the settings forms read, and logs every + * method with its key, so a test can assert which key the host asked for. + * + * @param seed - the references and records it starts with. + * @returns the service and its call log. + */ +export const fakeCredentials = (seed: CredentialSeed = {}): FakeCredentials => { + const refs = new Map(Object.entries(seed.refs ?? {})); + const records = new Map(Object.entries(seed.records ?? {})); + const calls: CredentialCall[] = []; + const log = (method: string, key: string): void => { + calls.push([method, key]); + seed.sharedLog?.push(`${method}:${key}`); + }; + const service = { + resolve: async (ref: string) => { + log("resolve", ref); + const value = refs.get(ref); + return value === undefined || value.length === 0 + ? undefined + : { source: "fake", value }; + }, + describe: async (ref: string) => { + log("describe", ref); + const configured = (refs.get(ref) ?? "").length > 0; + return configured + ? { configured, source: "fake", writable: true } + : { configured, writable: true }; + }, + set: async (ref: string, value: string) => { + log("set", ref); + refs.set(ref, value); + }, + unset: async (ref: string) => { + log("unset", ref); + refs.delete(ref); + }, + readRecord: async (key: string) => { + log("readRecord", key); + return records.get(key); + }, + describeRecord: async (key: string) => { + log("describeRecord", key); + const record = records.get(key) as { kind?: string } | undefined; + return record === undefined + ? { configured: false, writable: true } + : { configured: true, kind: record.kind, writable: true }; + }, + listRecords: async () => { + log("listRecords", "*"); + return [...records.entries()].map(([key, record]) => ({ + key, + kind: (record as { kind?: string }).kind, + })); + }, + modifyRecord: async ( + key: string, + mutate: (current: unknown) => Promise + ) => { + log("modifyRecord", key); + const next = await mutate(records.get(key)); + if (next !== undefined) { + records.set(key, next); + } + return records.get(key); + }, + deleteRecord: async (key: string) => { + log("deleteRecord", key); + records.delete(key); + }, + }; + return { + calls, + service: service as unknown as FakeCredentials["service"], + }; +}; + +/** The fake launch environment, with the names it was asked for. */ +export interface FakeLaunchEnvironment { + readonly reads: string[]; + readonly snapshot: { + readonly get: ( + name: string + ) => { value: string; source: "process" } | undefined; + readonly getFrom: ( + name: string, + sources: readonly string[] + ) => { value: string; source: "process" } | undefined; + }; +} + +/** + * A launch-environment snapshot over a plain record, in the shape + * `launchEnvironmentOf` hands back (`get`, `getFrom`). It stands in for the + * launcher's snapshot so the host never falls back to `process.env`. + * + * @param values - the variables the "process" layer holds. + * @param sharedLog - a log shared with {@link fakeCredentials}; each read + * appends `env:`. + * @returns the snapshot and the log of names read. + */ +export const fakeLaunchEnvironment = ( + values: Readonly> = {}, + sharedLog?: string[] +): FakeLaunchEnvironment => { + const reads: string[] = []; + const lookup = (key: string) => { + reads.push(key); + sharedLog?.push(`env:${key}`); + const value = values[key]; + return value === undefined + ? undefined + : { source: "process" as const, value }; + }; + return { + reads, + snapshot: { + get: lookup, + getFrom: (key, sources) => + sources.includes("process") ? lookup(key) : undefined, + }, + }; +}; + +// --------------------------------------------------------------------------- +// The wire +// --------------------------------------------------------------------------- + +/** One request the recording fetch saw. Credential header VALUES are never kept. */ +export interface RecordedRequest { + readonly url: string; + readonly method: string; + readonly model: string | undefined; + /** Lower-cased names; a credential header's value is replaced by its length. */ + readonly headers: Readonly>; +} + +/** The hosts the recording fetch answers; everything else is refused. */ +const GATEWAY_HOSTS = new Set(["opencode.ai", "models.opencode.ai"]); + +/** Headers whose value is a secret: recorded by length only. */ +const SECRET_HEADERS = new Set([ + "authorization", + "x-api-key", + "x-goog-api-key", +]); + +/** The 401 every gateway request gets unless the caller supplies a responder. */ +export const offlineAuthResponse = (): Response => + Response.json( + { error: { message: "probe: offline", type: "AuthError" } }, + { status: 401 } + ); + +/** A fetch replacement and what it recorded. */ +export interface RecordingFetch { + readonly fetch: typeof fetch; + readonly requests: RecordedRequest[]; +} + +const urlOf = (input: RequestInfo | URL): string => { + if (typeof input === "string") { + return input; + } + return input instanceof URL ? input.href : input.url; +}; + +const recordHeaders = ( + input: RequestInfo | URL, + init: RequestInit | undefined +): Record => { + const merged = new Headers(input instanceof Request ? input.headers : {}); + for (const [key, value] of new Headers(init?.headers ?? {})) { + merged.set(key, value); + } + const out: Record = {}; + for (const [key, value] of merged) { + out[key] = SECRET_HEADERS.has(key) ? value.length : value; + } + return out; +}; + +const bodyTextOf = async ( + input: RequestInfo | URL, + init: RequestInit | undefined +): Promise => { + if (typeof init?.body === "string") { + return init.body; + } + if (init?.body instanceof Uint8Array) { + return new TextDecoder().decode(init.body); + } + if (input instanceof Request) { + try { + return await input.clone().text(); + } catch { + return undefined; + } + } + return undefined; +}; + +/** + * A fetch that records every gateway request and answers it offline. + * + * @param options.respond - the answer for a gateway request (default: a 401). + * @returns the fetch and its log. + */ +export const recordingFetch = ( + options: { + readonly respond?: (request: RecordedRequest) => Response; + } = {} +): RecordingFetch => { + const requests: RecordedRequest[] = []; + const respond = options.respond ?? offlineAuthResponse; + const replacement = async ( + input: RequestInfo | URL, + init?: RequestInit + ): Promise => { + const url = urlOf(input); + let host = ""; + try { + ({ hostname: host } = new URL(url)); + } catch { + host = ""; + } + if (!GATEWAY_HOSTS.has(host)) { + throw new TypeError( + `host harness: refused a request to ${host || "a non-URL input"}` + ); + } + const text = await bodyTextOf(input, init); + let model: string | undefined; + try { + const parsed: unknown = JSON.parse(text ?? ""); + const candidate = (parsed as { model?: unknown } | null)?.model; + model = typeof candidate === "string" ? candidate : undefined; + } catch { + model = undefined; + } + const request: RecordedRequest = { + headers: recordHeaders(input, init), + method: ( + init?.method ?? (input instanceof Request ? input.method : "GET") + ).toUpperCase(), + model, + url, + }; + requests.push(request); + return respond(request); + }; + return { fetch: replacement as typeof fetch, requests }; +}; + +// --------------------------------------------------------------------------- +// The S0 fixture +// --------------------------------------------------------------------------- + +/** + * The profile run S0 measured: on each plane one known model per protocol plus + * the models pi 1.1.0 lacks, and a sibling pi-ai route that is not a plane. + * `deepseek`'s base is loopback port 9 (discard), so nothing reaches it. + */ +export const S0_PROFILE = { + deepseek: { + apiKeyEnv: "DEEPSEEK_API_KEY", + baseURL: "http://127.0.0.1:9/v1", + }, + opencode: { + apiKeyEnv: "OPENCODE_API_KEY", + models: [ + { id: "deepseek-v4-pro" }, + { id: "gpt-5.4" }, + { id: "claude-haiku-5-5" }, + { id: "gemini-3-flash" }, + { contextWindow: 400_000, id: "gpt-5.3-codex-spark", name: "My Spark" }, + ], + }, + "opencode-go": { + apiKeyEnv: "OPENCODE_GO_API_KEY", + models: [ + { id: "deepseek-v4-pro" }, + { id: "muse-spark-1.2-contributor" }, + { id: "minimax-m3" }, + { id: "gemini-4-argon-contributor" }, + { id: "minimax-m3.1-flash" }, + ], + }, +} as const; + +/** + * A DECLARED id that neither pi 1.1.0 nor the plugin's catalog describes (G1's + * second class, beside the gap ids the plugin's catalog still describes). + */ +export const UNKNOWN_ID = "not-in-any-catalog"; + +/** + * The characterization profile: {@link S0_PROFILE} with {@link UNKNOWN_ID} + * declared LAST on each plane, so the strict check still names each plane's + * stored gap id first. The contract tests keep plain S0, whose rows the + * declared unknown id would only blur. + */ +export const CHARACTERIZED_PROFILE = { + ...S0_PROFILE, + opencode: { + ...S0_PROFILE.opencode, + models: [...S0_PROFILE.opencode.models, { id: UNKNOWN_ID }], + }, + "opencode-go": { + ...S0_PROFILE["opencode-go"], + models: [...S0_PROFILE["opencode-go"].models, { id: UNKNOWN_ID }], + }, +} as const; + +/** The plane routes pi-ai serves today. */ +export const PLANES = ["opencode", "opencode-go"] as const; +export type Plane = (typeof PLANES)[number]; + +/** A fresh, mutable copy of {@link S0_PROFILE}. */ +export const s0Providers = (): Record> => + structuredClone(S0_PROFILE) as unknown as Record< + string, + Record + >; + +/** A fresh, mutable copy of {@link CHARACTERIZED_PROFILE}. */ +export const characterizedProviders = (): Record< + string, + Record +> => + structuredClone(CHARACTERIZED_PROFILE) as unknown as Record< + string, + Record + >; + +/** The characterized model ids per plane, in declaration order. */ +export const CHARACTERIZED_MODELS: Readonly> = + { + opencode: CHARACTERIZED_PROFILE.opencode.models.map((model) => model.id), + "opencode-go": CHARACTERIZED_PROFILE["opencode-go"].models.map( + (model) => model.id + ), + }; + +/** + * How a characterization row is tagged, so a later phase flips named rows: + * `KNOWN_DEFECT` is wrong today and a phase fixes it, `INTENDED_CHANGE` is + * right today and a phase changes it on purpose, `BASELINE` must survive. + */ +export type RowTag = "BASELINE" | "INTENDED_CHANGE" | "KNOWN_DEFECT"; + +/** + * The three facets a row records, each tagged on its own: one model can be + * right on the wire and wrong in validation (Zen gpt-5.3-codex-spark is). + * `info` is `context` + `efforts`, `callConfig` the verdict, `wire` is + * `sent` + `finish`. + */ +export type RowFacet = "callConfig" | "info" | "wire"; + +/** What TODAY's plugin does with one characterized model on pi 1.1.0, as measured. */ +export interface TodayRow { + readonly tags: Readonly>; + /** + * `resolveModelInfo`: its context window and effort ids in order, or the + * error code it rejects with. + */ + readonly info: + | { readonly context: number; readonly efforts: readonly string[] } + | { readonly error: string }; + /** `resolveCallConfig`: `"ok"`, or the error code it rejects with. */ + readonly callConfig: "ok" | "INVALID_CONFIG"; + /** The gateway URLs one turn sent, in order (the catalog read excluded). */ + readonly sent: readonly string[]; + /** The code on the turn's final `finish` chunk. */ + readonly finish: string; +} + +const ZEN = "https://opencode.ai"; + +/** Shorthand for a row's facet tags. */ +const tags = ( + info: RowTag, + callConfig: RowTag, + wire: RowTag +): Readonly> => ({ callConfig, info, wire }); + +const B = "BASELINE"; +const D = "KNOWN_DEFECT"; +const I = "INTENDED_CHANGE"; + +/** + * TODAY's plugin (hidden routes, listing patch, adapter decoration and all) on + * pi 1.1.0 with {@link CHARACTERIZED_PROFILE} and a key stored for each plane, + * measured in tier B with the published host's apply order (`plugin-first`, + * see {@link ApplyOrder}) and shared with the binary e2e so the two cannot + * drift. Each facet carries its own tag, and the characterization generates + * one test per row and facet titled with it: P-R1 flips the `KNOWN_DEFECT` and + * `INTENDED_CHANGE` facets its PR body names, and no `BASELINE` facet moves — + * except that P-R2's measured family flip, if a `modern` cell fails, may move + * the endpoint-family part of a `BASELINE` wire facet, re-tagging each one it + * moves and naming it in its PR body. + * + * `context` and `efforts` are copied from the generated catalog + * (`src/catalog-data.ts`), which the nightly refresh rewrites. A fixture-sanity + * case in `test/host-characterization.test.ts` compares each row's context AND + * effort ids to the bundled catalog's derivation, so a refresh that moves a + * fixture model fails there by name — update the row here. A phase that + * moves a non-`BASELINE` `info` facet off the catalog on purpose (D17's user + * row, D1's toggle efforts) adds that row's key to the case's + * `INFO_NOT_FROM_CATALOG` set in the same diff. The binary e2e + * reads the same table, but its job runs on a PR only when this file is in + * its path scope (`.github/workflows/ci.yml`, `e2e-host`), which it is. + */ +export const S0_TODAY = { + /** `listModels` before any turn: the plugin sees no key, so it hides paid models. */ + keylessListing: { opencode: [], "opencode-go": [] }, + /** `listModels` once a turn captured a key: pi-resolved ids only. */ + keyedListing: { + opencode: [ + "deepseek-v4-pro", + "gpt-5.4", + "claude-haiku-5-5", + "gemini-3-flash", + ], + "opencode-go": [ + "deepseek-v4-pro", + "muse-spark-1.2-contributor", + "minimax-m3", + ], + }, + rows: { + "opencode/deepseek-v4-pro": { + callConfig: "ok", + info: { context: 1_000_000, efforts: ["high", "max"] }, + finish: "AUTH", + sent: [`${ZEN}/zen/v1/chat/completions`], + tags: tags(B, B, I), + }, + "opencode/gpt-5.4": { + callConfig: "ok", + info: { context: 1_050_000, efforts: ["low", "medium", "high", "xhigh"] }, + finish: "AUTH", + sent: [`${ZEN}/inference/openai/v1/responses`], + tags: tags(B, B, B), + }, + "opencode/claude-haiku-5-5": { + callConfig: "ok", + info: { + context: 1_000_000, + efforts: ["low", "medium", "high", "xhigh", "max"], + }, + finish: "AUTH", + sent: [`${ZEN}/inference/anthropic/v1/messages?beta=true`], + tags: tags(B, B, B), + }, + "opencode/gemini-3-flash": { + callConfig: "ok", + info: { + context: 1_048_576, + efforts: ["minimal", "low", "medium", "high"], + }, + finish: "PI_AI_ERROR", + sent: [ + `${ZEN}/inference/google/v1beta/models/gemini-3-flash:streamGenerateContent?alt=sse`, + ], + tags: tags(B, B, B), + }, + // The user's row (name "My Spark", contextWindow 400000) is ignored. + "opencode/gpt-5.3-codex-spark": { + callConfig: "INVALID_CONFIG", + info: { context: 128_000, efforts: ["low", "medium", "high", "xhigh"] }, + finish: "AUTH", + sent: [`${ZEN}/inference/openai/v1/responses`], + tags: tags(D, D, B), + }, + "opencode/not-in-any-catalog": { + callConfig: "INVALID_CONFIG", + finish: "INVALID_CONFIG", + info: { error: "INVALID_CONFIG" }, + sent: [], + tags: tags(D, D, D), + }, + "opencode-go/deepseek-v4-pro": { + callConfig: "ok", + info: { context: 1_000_000, efforts: ["high", "max"] }, + finish: "AUTH", + sent: [`${ZEN}/zen/go/v1/chat/completions`], + tags: tags(B, B, I), + }, + "opencode-go/muse-spark-1.2-contributor": { + callConfig: "ok", + info: { + context: 1_048_576, + efforts: ["minimal", "low", "medium", "high", "xhigh"], + }, + finish: "AUTH", + sent: [`${ZEN}/inference/go/openai/v1/responses`], + tags: tags(B, B, B), + }, + // A toggle-only model offered Minimal/Xhigh/Max (D1). + "opencode-go/minimax-m3": { + callConfig: "ok", + info: { + context: 1_000_000, + efforts: ["minimal", "low", "medium", "high", "xhigh", "max"], + }, + finish: "AUTH", + sent: [`${ZEN}/inference/go/anthropic/v1/messages?beta=true`], + tags: tags(D, B, B), + }, + "opencode-go/gemini-4-argon-contributor": { + callConfig: "INVALID_CONFIG", + info: { context: 1_048_576, efforts: ["low", "medium", "high"] }, + finish: "PI_AI_ERROR", + sent: [ + `${ZEN}/inference/go/google/v1beta/models/gemini-4-argon-contributor:streamGenerateContent?alt=sse`, + ], + tags: tags(B, D, B), + }, + "opencode-go/minimax-m3.1-flash": { + callConfig: "INVALID_CONFIG", + info: { + context: 1_000_000, + efforts: ["low", "medium", "high", "xhigh", "max"], + }, + finish: "INVALID_CONFIG", + sent: [], + tags: tags(B, D, D), + }, + "opencode-go/not-in-any-catalog": { + callConfig: "INVALID_CONFIG", + finish: "INVALID_CONFIG", + info: { error: "INVALID_CONFIG" }, + sent: [], + tags: tags(D, D, D), + }, + } satisfies Record, + /** + * What appending a new id to `opencode` answers while the plane stores a gap + * id: the strict check names the STORED gap id, whatever is being saved. + */ + saveBlockedByStoredGap: + /provider "opencode" model "gpt-5\.3-codex-spark" needs an api/u, + /** What saving a catalog-unknown id answers once the plane stores no gap. */ + saveRejectedUnknown: + /provider "opencode" model "not-a-real-model" needs an api/u, +} as const; + +/** The id the save rows try to add: unknown to every catalog, and not declared. */ +export const SAVE_CANDIDATE = "not-a-real-model"; + +/** + * The ids on `opencode` that pi 1.1.0 cannot place: the stored gap id and the + * declared unknown id. The save sequence (in-process, and the probe's + * `PROBE_SAVE_DROP`) removes them before saving {@link SAVE_CANDIDATE}. + */ +export const ZEN_UNDESCRIBED = ["gpt-5.3-codex-spark", UNKNOWN_ID] as const; + +/** + * TODAY's rows for an apply order. `S0_TODAY.rows` is the `plugin-first` + * table; with `pi-ai-first` the adapter decoration wraps pi-ai's plane adapter + * and exactly one facet moves — Go gemini-4-argon-contributor validates — + * measured in `test/host-characterization.test.ts`. The binary e2e picks the + * table from the order its probe INFERRED — whether the decoration wrapped + * pi-ai's plane adapter, which is what the rows depend on — because on a real + * host the order is a race (F17) the suite must not pin. + * + * @param order - the apply order, as inferred from the decoration. + * @returns the rows, keyed `/`. + */ +export const todayRowsFor = ( + order: ApplyOrder +): Readonly> => { + if (order === "plugin-first") { + return S0_TODAY.rows; + } + // Exhaustive on purpose: a probe JSON with a missing or renamed field reads + // `undefined` here, and must not fall through to the pi-ai-first table. + if (order !== "pi-ai-first") { + throw new Error(`todayRowsFor: unknown apply order ${String(order)}`); + } + return { + ...S0_TODAY.rows, + "opencode-go/gemini-4-argon-contributor": { + ...S0_TODAY.rows["opencode-go/gemini-4-argon-contributor"], + callConfig: "ok", + }, + }; +}; + +/** + * Every {@link ApplyOrder}, for iterating {@link todayRowsFor}'s tables. Built + * from a `Record` keyed by the union, so a member added to `ApplyOrder` and + * missing here fails to compile rather than going unchecked. + */ +export const APPLY_ORDERS = Object.keys({ + "pi-ai-first": true, + "plugin-first": true, +} satisfies Record) as readonly ApplyOrder[]; + +/** + * What appending a new id to `opencode` answers while the plane stores ANY of + * {@link ZEN_UNDESCRIBED}, whichever one the installed pi cannot describe + * first. The pinned pi always names `gpt-5.3-codex-spark` + * (`S0_TODAY.saveBlockedByStoredGap`); a floating pi that learned it names the + * next one, so the floating cell asserts this instead. + */ +export const SAVE_BLOCKED_BY_ANY_UNDESCRIBED = new RegExp( + `provider "opencode" model "(?:${ZEN_UNDESCRIBED.map((id) => + id.replaceAll(".", String.raw`\.`) + ).join("|")})" needs an api`, + "u" +); + +const PI_DATA = new URL( + "../../node_modules/@earendil-works/pi-ai/dist/providers/data/", + import.meta.url +); + +const piData = (plane: Plane): Record> => + JSON.parse( + readFileSync( + new URL( + plane === "opencode" ? "opencode.json" : "opencode-go.json", + PI_DATA + ), + "utf8" + ) + ) as Record>; + +/** + * The protocol the installed pi's catalog files a model under on a plane, or + * `undefined` when pi does not know it. The data is keyed by protocol, then + * `chat:`, so a fixture can check which of its ids are gaps on pi 1.1.0. + * + * @param plane - the plane route. + * @param id - the model id. + * @returns the protocol, or `undefined` for a gap. + */ +export const piKnows = (plane: Plane, id: string): string | undefined => + Object.entries(piData(plane)).find(([, models]) => + Object.hasOwn(models, `chat:${id}`) + )?.[0]; + +// --------------------------------------------------------------------------- +// Waterfall helpers +// --------------------------------------------------------------------------- + +/** + * A fresh deep copy of a raw pi-ai config without the two plane entries. Pure: + * it never touches its input, so it is safe on a frozen one. + * + * @param raw - the raw config, `{ providers }`. + * @returns the copy. + */ +export const withoutPlanes = (raw: T): T => { + const copy = structuredClone(raw) as { providers?: Record }; + if (copy.providers !== undefined && copy.providers !== null) { + delete copy.providers.opencode; + delete copy.providers["opencode-go"]; + } + return copy as T; +}; + +/** Deep-freeze a value and everything under it. */ +export const deepFreeze = (value: T): T => { + if (value !== null && typeof value === "object") { + for (const child of Object.values(value)) { + deepFreeze(child); + } + Object.freeze(value); + } + return value; +}; + +/** One `internal/config` run the spy saw. */ +export interface WaterfallRecord { + readonly self: Fiber; + readonly raw: unknown; + /** `raw === this._config` at the moment the listener ran (F14). */ + readonly isCommit: boolean; + readonly order: number; +} + +/** + * Record every `internal/config` run for one fiber, from a listener prepended + * on `ctx` (so it is outermost among those registered before it). + * + * @param ctx - the context to register on (the root, or another plugin's). + * @param fiber - the fiber whose runs are kept; omit to keep every run. + * @returns the records, and the disposer. + */ +export const waterfallSpy = ( + ctx: Context, + fiber?: () => Fiber | undefined +): { records: WaterfallRecord[]; dispose: ReturnType } => { + const records: WaterfallRecord[] = []; + const dispose = ctx.on( + "internal/config", + function recordRun(this: Fiber, raw: unknown, next: () => unknown) { + const target = fiber?.(); + if (target === undefined || this === target) { + records.push({ + isCommit: raw === (this as unknown as { _config: unknown })._config, + order: records.length, + raw, + self: this, + }); + } + return next(); + }, + { prepend: true } + ); + return { dispose, records }; +}; + +/** + * Poll the registry until every route is present: the plugin mounts its + * hidden routes asynchronously (`mountInternalRoutes` in + * `src/lifecycle.ts`). + * + * @param llm - the `llm` service. + * @param ids - the routes to wait for. + * @param deadlineMs - how long to wait before failing. The default sits well + * inside the 30 s a tier-B hook gets for the boot AND this wait, so a route + * that never registers fails by name rather than as a hook timeout. + */ +export const waitForRoutes = async ( + llm: HostLlm, + ids: readonly string[], + deadlineMs = 10_000 +): Promise => { + const until = Date.now() + deadlineMs; + for (;;) { + const present = adaptersOf(llm); + const missing = ids.filter((id) => !present.has(id)); + if (missing.length === 0) { + return; + } + if (Date.now() > until) { + throw new Error( + `host harness: routes never registered: ${missing.join(", ")}` + ); + } + await new Promise((resolve) => { + setTimeout(resolve, 20); + }); + } +}; + +/** One drained `llm.stream` call: the chunk types and the final finish. */ +export interface DrainedStream { + readonly types: string[]; + readonly finish: + | { readonly kind: string | undefined; readonly code: string | undefined } + | undefined; + readonly error: string | undefined; +} + +/** + * Drive one turn through the host's own stream path and keep what it said. + * + * @param llm - the `llm` service. + * @param provider - the route. + * @param model - the model id. + * @returns the drained stream. + */ +export const drainStream = async ( + llm: HostLlm, + provider: string, + model: string +): Promise => { + const types: string[] = []; + let finish: DrainedStream["finish"]; + let failed: string | undefined; + try { + for await (const chunk of llm.stream({ + messages: [{ content: [{ text: "hi", type: "text" }], role: "user" }], + model, + provider, + signal: AbortSignal.timeout(10_000), + } as never)) { + const typed = chunk as { + type: string; + reason?: { kind?: string; failure?: { code?: string } }; + }; + types.push(typed.type); + if (typed.type === "finish") { + finish = { + code: typed.reason?.failure?.code, + kind: typed.reason?.kind, + }; + } + } + } catch (error) { + const failure = error as { code?: unknown; message?: unknown }; + failed = `${typeof failure.code === "string" ? `${failure.code} ` : ""}${String(failure.message ?? error)}`; + } + return { error: failed, finish, types }; +}; + +// --------------------------------------------------------------------------- +// Tier A +// --------------------------------------------------------------------------- + +/** What {@link mountTierA} takes. */ +export interface TierAOptions { + /** pi-ai's `providers` dict. */ + readonly providers: Record; + /** pi-ai's whole raw config, used as given instead of `{ providers }` (a frozen input). */ + readonly piConfig?: object; + /** + * Mount this plugin too, with this row config. Stub `fetch` first (its fetch + * patch captures `globalThis.fetch` at effect time). Tier A has no loader, + * so the plugin cannot find pi-ai's entry: it applies and patches fetch, but + * mounts NO hidden route and no Google bridge: the bridge's `PiAiAdapter` + * comes from the loader's pi-ai entry (`loadPiAiAdapterHost`), which tier A + * cannot answer. `test/host-harness.test.ts` pins both — the stand-down line + * and empty hidden routes, and `loadPiAiAdapterHost` answering `undefined`. + * Characterize the real plugin in tier B. + */ + readonly plugin?: Record; + readonly credentials?: FakeCredentials; + readonly launchEnvironment?: FakeLaunchEnvironment; + /** Runs on the root before pi-ai mounts (a listener to register first). */ + readonly before?: (ctx: Context) => void; +} + +/** A mounted tier A. */ +export interface TierA { + readonly ctx: Context; + readonly llm: HostLlm; + readonly piFiber: Fiber; + readonly pluginFiber: Fiber | undefined; + readonly credentials: FakeCredentials; + readonly launchEnvironment: FakeLaunchEnvironment; + readonly dispose: () => Promise; +} + +/** + * A bare `Context` with `LlmRuntime` and pi-ai mounted as plugins. + * + * @param options - the providers, and the optional plugin config and fakes. + * @returns the mount; the caller disposes it. + */ +export const mountTierA = async (options: TierAOptions): Promise => { + const ctx = new Context(); + const credentials = options.credentials ?? fakeCredentials(); + const launchEnvironment = + options.launchEnvironment ?? fakeLaunchEnvironment(); + try { + ctx.provide("credentials", credentials.service); + ctx.provide("launchEnvironment", launchEnvironment.snapshot); + await ctx.plugin(LlmRuntime); + options.before?.(ctx); + const piFiber: Fiber = await ctx.plugin( + LlmPiAi as never, + (options.piConfig ?? { providers: options.providers }) as never + ); + let pluginFiber: Fiber | undefined; + if (options.plugin !== undefined) { + pluginFiber = await ctx.plugin( + { apply, Config, inject, name } as never, + options.plugin as never + ); + } + return { + credentials, + ctx, + dispose: async () => { + await ctx.fiber.dispose(); + }, + launchEnvironment, + llm: ctx.llm, + piFiber, + pluginFiber, + }; + } catch (error) { + await ctx.fiber.dispose(); + throw error; + } +}; + +// --------------------------------------------------------------------------- +// Tier B +// --------------------------------------------------------------------------- + +/** The modules the stub loader resolves, by package name. */ +const HOST_MODULES: ReadonlyMap = new Map([ + ["@deepseek-ai/dsh-llm", LlmRuntime], + [PI_AI_PACKAGE, LlmPiAi], + ["@deepseek-ai/dsh-config-editor", ConfigEditor], + ["@deepseek-ai/dsh-settings", Settings], + [name, { apply, Config, inject, name }], +]); + +/** + * The loader's module seam, answering only the five host rows. Any other + * import is a composition this harness did not intend, so it throws. A value + * may be a promise, which is how {@link orderedModules} holds one row back. + * + * @param modules - the namespaces (or promises of them) by specifier. + * @returns the `ModuleLoaderV2` to put on `root.loader.internal`. + */ +export const stubModuleLoader = ( + modules: ReadonlyMap = HOST_MODULES +): ModuleLoaderV2 => ({ + getOrCreateModuleJob(): never { + throw new Error("unexpected module job creation"); + }, + import: async (specifier: string) => { + if (!modules.has(specifier)) { + throw new Error(`unexpected Loader import: ${specifier}`); + } + return await modules.get(specifier); + }, + load(): never { + throw new Error("unexpected module load"); + }, + loadCache: new Map() as never, + register(): never { + throw new Error("unexpected module hook registration"); + }, + resolveSync(): never { + throw new Error("unexpected synchronous module resolution"); + }, + version: "v2", +}); + +/** + * Which of pi-ai and this plugin APPLIES first. The loader starts rows in + * parallel (F17), so on a real host the order is whatever module loading makes + * it. Measured on the published binary: the plugin applies first, about 70 ms + * before pi-ai registers its routes, because pi-ai's module is the heavier + * import. The stub loader resolves every import at once, so the harness makes + * the order explicit instead of inheriting row order. + */ +export type ApplyOrder = "pi-ai-first" | "plugin-first"; + +/** The fiber states a row settles in: ACTIVE, FAILED, DISPOSED (cordis `FiberState`). */ +const SETTLED_STATES: ReadonlySet = new Set([2, 3, 4]); + +/** {@link orderedModules}' answer: the modules, and the hook that arms the fallback release. */ +interface OrderedModules { + readonly modules: ReadonlyMap; + /** Register, on the root, the release for a first row that never applies. */ + readonly watch: (root: Context) => void; +} + +/** + * The host modules with one of the two rows held back until the other's + * `apply` has returned. + * + * A first row can also settle WITHOUT applying: cordis resolves a fiber's + * config before it runs `apply` (`fiber.ts` `_reload`), so a config the + * schema rejects leaves the fiber FAILED and `apply` never called. The held + * import would then never resolve, the loader would wait on it for ever, and + * `boot()` with it. So `watch` also releases the held row as soon as the first + * row's fiber reports a settled state (`internal/status`), and the boot + * finishes with that fiber FAILED — its error readable through + * `fiber.await()`. A row that never leaves PENDING (a required service that + * never appears) still holds the second one; no row here injects one. + * + * @param order - which applies first. + * @returns the modules for {@link stubModuleLoader}, and the fallback hook. + */ +const orderedModules = (order: ApplyOrder): OrderedModules => { + const { promise: released, resolve: release } = + Promise.withResolvers(); + const [first, second] = + order === "plugin-first" ? [name, PI_AI_PACKAGE] : [PI_AI_PACKAGE, name]; + const firstModule = HOST_MODULES.get(first) as { + apply: (ctx: unknown, config: unknown) => unknown; + }; + const held = async (): Promise => { + await released; + return HOST_MODULES.get(second); + }; + const firstApply = (ctx: unknown, config: unknown): unknown => { + try { + return firstModule.apply(ctx, config); + } finally { + release(true); + } + }; + return { + modules: new Map([ + ...HOST_MODULES, + [first, { ...firstModule, apply: firstApply }], + [second, held()], + ]), + watch: (root) => { + root.on("internal/status", (fiber: Fiber) => { + // The registry keys a runtime by its plugin's `apply`, so this is the + // first row's fiber and no other. + if ( + fiber.runtime?.callback === firstApply && + SETTLED_STATES.has(fiber.state) + ) { + release(true); + } + }); + }, + }; +}; + +/** The bundle rows, in the ids the real host's `dsh-base` patch names them. */ +const BUNDLE_ROWS = [ + { id: "llm", name: "@deepseek-ai/dsh-llm" }, + { id: "llm-pi-ai", name: PI_AI_PACKAGE }, + { id: "config-editor", name: "@deepseek-ai/dsh-config-editor" }, + { id: "settings", name: "@deepseek-ai/dsh-settings" }, + { id: name, name }, +]; + +/** What {@link bootTierB} takes. */ +export interface TierBOptions { + /** pi-ai's `providers` dict, written to the profile patch. */ + readonly providers?: Record; + /** Mount this plugin's row (default `true`); host-only contracts leave it out. */ + readonly plugin?: boolean; + /** A profile row for this plugin's config; omitted, the shipped defaults apply. */ + readonly pluginConfig?: Record; + /** The profile patch as raw YAML text, replacing the generated one (for `!!js`). */ + readonly profilePatchYaml?: string; + readonly credentials?: FakeCredentials; + readonly launchEnvironment?: FakeLaunchEnvironment; + /** Runs on the root after the fakes are provided and before any row mounts. */ + readonly prepare?: (root: Context) => void; + /** + * Which of pi-ai and the plugin applies first (default `"plugin-first"`, the + * order measured on the published binary). Ignored without the plugin row. + */ + readonly order?: ApplyOrder; +} + +/** A booted tier B. */ +export interface TierB { + readonly ctx: Context; + readonly home: string; + readonly profile: ProfileContext; + readonly llm: HostLlm; + readonly piEntry: Entry; + readonly piFiber: Fiber; + /** The plugin's entry; `undefined` when the boot left the row out. */ + readonly pluginEntry: Entry | undefined; + readonly credentials: FakeCredentials; + readonly dispose: () => Promise; +} + +/** The keys the plugin reads from the environment directly. */ +export const AMBIENT_KEYS = [ + "OPENCODE_API_KEY", + "OPENCODE_GO_API_KEY", +] as const; + +/** + * Blank the keys the plugin reads directly (`hasPlaneCredential` in + * `src/lifecycle.ts`) for the life of one boot, so a contributor's exported + * key cannot turn the keyless rows into keyed ones. An empty value is what the + * plugin reads as "no key". Only a key that IS set is touched, and the + * returned restore puts back exactly what was there. + * + * @returns the restore, which the boot's dispose (and its failure path) runs. + */ +const blankAmbientKeys = (): (() => void) => { + const saved = AMBIENT_KEYS.filter( + (key) => (process.env[key] ?? "").length > 0 + ).map((key) => [key, process.env[key]] as const); + for (const [key] of saved) { + process.env[key] = ""; + } + return () => { + for (const [key, value] of saved) { + process.env[key] = value; + } + }; +}; + +/** The profile patch the S0-style boots write, as JSON text (valid YAML). */ +const profilePatchText = ( + providers: Record | undefined, + pluginConfig: Record | undefined +): string => + `${JSON.stringify( + [ + ...(providers === undefined + ? [] + : [{ config: { providers }, id: "llm-pi-ai" }]), + ...(pluginConfig === undefined + ? [] + : [{ config: pluginConfig, id: name }]), + ], + null, + 2 + )}\n`; + +/** + * Boot the host the way the harness's own composition tests do, with this + * plugin's row mounted as the real host mounts it. + * + * @param options - the providers, the plugin row and the fakes. + * @returns the booted host; the caller disposes it. + */ +export const bootTierB = async (options: TierBOptions = {}): Promise => { + const credentials = options.credentials ?? fakeCredentials(); + const launchEnvironment = + options.launchEnvironment ?? fakeLaunchEnvironment(); + const home = realpathSync(mkdtempSync(join(tmpdir(), "dsh-opencode-host-"))); + const restoreKeys = blankAmbientKeys(); + let ctx: Context | undefined; + try { + const dir = join(home, "profiles", "test"); + initProfile(dir, ["test-bundle"]); + const bundle = join(dir, "node_modules", "test-bundle"); + mkdirSync(bundle, { recursive: true }); + writeFileSync(join(home, "package.json"), '{"name":"test-installation"}\n'); + writeFileSync( + join(bundle, "package.json"), + JSON.stringify({ + dsh: { bundle: { patch: "cordis.patch.yml" } }, + name: "test-bundle", + version: "1.0.0", + }) + ); + writeFileSync( + join(bundle, "cordis.patch.yml"), + `${JSON.stringify( + [ + { + insert: BUNDLE_ROWS.filter( + (row) => options.plugin !== false || row.id !== name + ), + }, + ], + null, + 2 + )}\n` + ); + writeFileSync(join(dir, "cordis.yml"), "[]\n"); + const patchPath = join(dir, "cordis.patch.yml"); + writeFileSync( + patchPath, + options.profilePatchYaml ?? + profilePatchText(options.providers, options.pluginConfig) + ); + const profile: ProfileContext = { + cwd: home, + dir, + home, + installAnchor: join(home, "package.json"), + name: "test", + overlays: [], + patchPath, + startedBundles: ["test-bundle"], + telemetryDisabledEnv: undefined, + }; + ctx = await boot( + "dsh", + join(dir, "cordis.yml"), + readProfilePatches("dsh", profile), + (root) => { + root.provide("profileContext", profile); + root.provide("credentials", credentials.service); + root.provide("launchEnvironment", launchEnvironment.snapshot); + if (options.plugin === false) { + root.loader.internal = stubModuleLoader(HOST_MODULES); + } else { + const ordered = orderedModules(options.order ?? "plugin-first"); + ordered.watch(root); + root.loader.internal = stubModuleLoader(ordered.modules); + } + options.prepare?.(root); + } + ); + const booted = ctx; + const entries = booted.configEditor.entries(); + const entryFor = (id: string): Entry => { + const found = entries.find((entry) => entry.options.id === id); + if (found?.fiber === undefined) { + throw new Error(`host harness: no mounted entry ${id}`); + } + return found; + }; + const piEntry = entryFor("llm-pi-ai"); + const pluginEntry = options.plugin === false ? undefined : entryFor(name); + return { + credentials, + ctx: booted, + dispose: async () => { + try { + await booted.fiber.dispose(); + } finally { + rmSync(home, { force: true, recursive: true }); + restoreKeys(); + } + }, + home, + llm: booted.llm, + piEntry, + get piFiber() { + return piEntry.fiber!; + }, + pluginEntry, + profile, + }; + } catch (error) { + try { + await ctx?.fiber.dispose(); + } finally { + rmSync(home, { force: true, recursive: true }); + restoreKeys(); + } + throw error; + } +}; + +/** Only the two plane keys stored: what a user who pasted them into Settings has. */ +export const planeKeysOnly = (): FakeCredentials => + fakeCredentials({ + refs: { OPENCODE_API_KEY: "zen-test", OPENCODE_GO_API_KEY: "go-test" }, + }); + +/** + * session-controller's `hasProviderApiKey` (`api/session-controller/src/catalog.ts:97-116`), + * restated line for line from the real parts, because the package does not + * export it (its `exports` map reaches `SessionController` and + * `buildModelCatalog` only). `initializeDefaultModel` (`index.ts:320-331`) + * calls it after a Sign-in and, when it answers false, replaces the user's + * default model with the account's first one. It answers the FIRST + * configurable provider whose profile, read from `settings.describe`'s VALUE + * at the provider's `settingsPath`, names an `apiKeyEnv` the credentials + * service reports as configured — so it is the reader F20 changes. + * + * @param tier - the booted host. + * @returns that provider's id (`true` upstream), or `undefined` (`false`). + */ +export const providerWithApiKey = async ( + tier: TierB +): Promise => { + const describeRef = tier.credentials.service.describe as unknown as ( + ref: string + ) => Promise<{ configured: boolean }>; + const namespaces = tier.ctx.settings.describe({ redactSecrets: true }); + for (const provider of tier.llm.listConfigurableProviders()) { + if (provider.provider === "deepseek-account") { + continue; + } + let profile: unknown = namespaces.find( + (namespace) => namespace.ns === provider.settingsNs + )?.value; + for (const key of provider.settingsPath) { + profile = + typeof profile === "object" && profile !== null + ? Reflect.get(profile, key) + : undefined; + } + if (typeof profile !== "object" || profile === null) { + continue; + } + const ref: unknown = Reflect.get(profile, "apiKeyEnv"); + if (typeof ref === "string" && ref.length > 0) { + const described = await describeRef(ref); + if (described.configured) { + return provider.provider; + } + } + } + return undefined; +}; diff --git a/test/support/keyless-matrix.ts b/test/support/keyless-matrix.ts new file mode 100644 index 00000000..d83d6235 --- /dev/null +++ b/test/support/keyless-matrix.ts @@ -0,0 +1,295 @@ +/** + * The keyless 401-vs-404 probe matrix: which gateway paths still exist. + * + * Two planes × four protocols × two endpoint families, one representative + * PAID model per plane and protocol, every request sent WITHOUT a key. The gateway + * authenticates before it routes a model, so a keyless request tells the paths + * apart without spending anything: a 404 means the path is gone, a 401/403 + * that it exists, a 410 or a 400 "Model is unavailable" that the model is. + * `test/e2e/keyless-matrix.e2e.ts` sends the cells; P-R2 decides the default + * family from what they answer. + * + * Pure: every function here is a function of its arguments — no fetch, no + * environment, no module state — so the cell table and the classifier are + * unit-tested offline (`test/keyless-matrix.test.ts`). There is no code path + * here that can set `authorization`, `x-api-key` or `x-goog-api-key`. + * + * @module test/support/keyless-matrix + */ +import { + ANTHROPIC_VERSION, + type CatalogModelSpec, + LEGACY_PLANE_API, + OPENCODE_FAMILY_API, + protocolFor, +} from "./src.ts"; + +export type MatrixPlane = "opencode" | "opencode-go"; +export type MatrixFamily = "legacy" | "modern"; + +/** The four protocols, in the order cells are produced. */ +export const MATRIX_PROTOCOLS = [ + "openai-completions", + "openai-responses", + "anthropic-messages", + "google-generative-ai", +] as const; +export type MatrixProtocol = (typeof MATRIX_PROTOCOLS)[number]; + +/** One probe: what to send, and to where. */ +export interface MatrixCell { + readonly plane: MatrixPlane; + readonly protocol: MatrixProtocol; + readonly family: MatrixFamily; + /** `undefined` when the plane's catalog has no model for the protocol. */ + readonly model: string | undefined; + readonly url: string; + readonly method: "POST"; + readonly headers: Readonly>; + /** `undefined` exactly when `model` is. */ + readonly body: Readonly> | undefined; +} + +/** A cell's name: ` `, as the live table prints it. */ +export const cellKey = ( + cell: Pick +): string => `${cell.plane} ${cell.protocol} ${cell.family}`; + +/** + * The cells TODAY's plugin sends turns to with the default endpoint (`auto`, + * an alias of `modern`), on both planes: completions on pi 1.1.0's LEGACY + * bases (`/zen(/go)/v1/chat/completions`, pi-ai's own plane route) and the + * three hidden routes' MODERN paths (Responses, Messages, Google). A path-gone + * verdict on one of these breaks a shipped model in the default + * configuration, so the live matrix asserts them. `test/keyless-matrix.test.ts` + * derives the same set from the URLs `S0_TODAY` measured (with the default + * endpoint), so the two cannot drift. + * + * Not covered: a user who sets `endpoint: legacy` dispatches Responses and + * Messages on both planes, and Zen's Google, to the LEGACY cells, which the + * matrix records for P-R2 and does not assert; P-R2 decides whether that + * configuration is guarded. Go's Google is the exception: it stays on + * `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL` in + * `src/google-bridge.ts`), which is the modern cell this set already holds. + */ +export const TODAY_DISPATCH: ReadonlySet = new Set( + (["opencode", "opencode-go"] as const).flatMap((plane) => [ + cellKey({ family: "legacy", plane, protocol: "openai-completions" }), + cellKey({ family: "modern", plane, protocol: "openai-responses" }), + cellKey({ family: "modern", plane, protocol: "anthropic-messages" }), + cellKey({ family: "modern", plane, protocol: "google-generative-ai" }), + ]) +); + +/** + * The cells P-R1 moves traffic to that nothing dispatches to yet: completions + * on the modern family (`/inference(/go)/openai/v1`). The live matrix asserts + * them too, as P-R1's forward guard, and labels them so in its failure. + */ +export const NEXT_DISPATCH: ReadonlySet = new Set( + (["opencode", "opencode-go"] as const).map((plane) => + cellKey({ family: "modern", plane, protocol: "openai-completions" }) + ) +); + +/** What a probe answered, as the matrix reads it. */ +export type KeylessVerdict = + | "model-gone" + | "no-model" + | "other" + | "path-exists" + | "path-gone"; + +/** The catalog SDK each protocol is served for, for the modern family table. */ +const SDK_FOR: Readonly> = { + "anthropic-messages": "@ai-sdk/anthropic", + "google-generative-ai": "@ai-sdk/google", + "openai-completions": "", + "openai-responses": "@ai-sdk/openai", +}; + +/** The path each body-addressed protocol appends to its base. */ +const BODY_PATH: Readonly< + Record, string> +> = { + "anthropic-messages": "/messages", + "openai-completions": "/chat/completions", + "openai-responses": "/responses", +}; + +/** + * The path a protocol appends to its base. Google names the model in the path + * and streams; the three others name it in the body. + */ +const suffixFor = ( + protocol: MatrixProtocol, + model: string | undefined +): string => + protocol === "google-generative-ai" + ? `/models/${model ?? ""}:streamGenerateContent?alt=sse` + : BODY_PATH[protocol]; + +/** + * The URL a cell probes. + * + * The Go plane's legacy Google cell has no `?alt=sse`: pi 1.1.0 ships no Go + * Google provider, so that path is the bare one the plan names + * (`/zen/go/v1/models/:streamGenerateContent`), not a request pi makes. + */ +export const matrixUrl = ( + plane: MatrixPlane, + protocol: MatrixProtocol, + family: MatrixFamily, + model: string | undefined +): string => { + if (family === "legacy") { + // The legacy family: one base per plane, every protocol under its `/v1`. + const url = `${LEGACY_PLANE_API[plane]}${suffixFor(protocol, model)}`; + return plane === "opencode-go" && protocol === "google-generative-ai" + ? url.replace("?alt=sse", "") + : url; + } + const table = OPENCODE_FAMILY_API[plane === "opencode-go" ? "go" : "zen"]; + return `${table[SDK_FOR[protocol]] ?? ""}${suffixFor(protocol, model)}`; +}; + +/** The smallest valid request body for a protocol, carrying the model. */ +export const matrixBody = ( + protocol: MatrixProtocol, + model: string +): Readonly> => { + if (protocol === "openai-responses") { + return { input: "hi", max_output_tokens: 1, model }; + } + if (protocol === "google-generative-ai") { + return { + contents: [{ parts: [{ text: "hi" }], role: "user" }], + generationConfig: { maxOutputTokens: 1 }, + }; + } + // Chat completions and Messages share the minimal shape. + return { + max_tokens: 1, + messages: [{ content: "hi", role: "user" }], + model, + }; +}; + +/** The only headers a cell carries: never a credential. */ +const headersFor = (protocol: MatrixProtocol): Record => + protocol === "anthropic-messages" + ? { + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json", + } + : { "content-type": "application/json" }; + +/** + * Whether a catalog row is a free model. A keyless probe must never pick one: + * the gateway serves an unmetered model WITHOUT a key (measured: `beam-free` + * answers `200` with a real completion), so its cell would spend a completion + * and read as "other" instead of telling the path apart, and a free-tier + * model answers a wrong endpoint with a 500. The id test is defence in depth + * for a slice whose `is_free` was not derived (`parseModelsDevCatalog` sets it + * from the id or a zero cost). + */ +export const isFreeSpec = (spec: CatalogModelSpec): boolean => + spec.is_free === true || /free/iu.test(spec.id); + +/** + * The representative model for a protocol: the first catalog row that speaks + * it, is not free, and is not deprecated. A paid model is the class the + * gateway answers with 401 before it routes, which is what the matrix reads. + */ +const representative = ( + specs: readonly CatalogModelSpec[], + protocol: MatrixProtocol, + isDeprecated: (id: string) => boolean +): string | undefined => + specs.find( + (spec) => + protocolFor(spec) === protocol && + !isFreeSpec(spec) && + !isDeprecated(spec.id) + )?.id; + +/** + * Build the sixteen cells. + * + * @param catalogs.zen - the Zen plane's catalog slice. + * @param catalogs.go - the Go plane's catalog slice. + * @param catalogs.isDeprecated - which ids the catalog retired (default: none). + * A slice from `parseModelsDevCatalog` has already dropped its deprecated + * rows, so the live matrix passes none; this guards an unparsed slice. + * @returns the cells, plane by plane, protocol by protocol, legacy first. + */ +export const keylessMatrix = (catalogs: { + readonly zen: readonly CatalogModelSpec[]; + readonly go: readonly CatalogModelSpec[]; + readonly isDeprecated?: (plane: MatrixPlane, id: string) => boolean; +}): MatrixCell[] => { + const cells: MatrixCell[] = []; + for (const plane of ["opencode", "opencode-go"] as const) { + const specs = plane === "opencode" ? catalogs.zen : catalogs.go; + for (const protocol of MATRIX_PROTOCOLS) { + const model = representative( + specs, + protocol, + (id) => catalogs.isDeprecated?.(plane, id) ?? false + ); + for (const family of ["legacy", "modern"] as const) { + cells.push({ + body: model === undefined ? undefined : matrixBody(protocol, model), + family, + headers: headersFor(protocol), + method: "POST", + model, + plane, + protocol, + url: matrixUrl(plane, protocol, family, model), + }); + } + } + } + return cells; +}; + +/** + * Whether a probe's answer is transient and worth one retry before it is + * classified: a rate limit, a server error, or no answer at all. The live + * matrix sends sixteen requests on every pull request, and one 429 must not + * turn the job red. + * + * @param status - the HTTP status, or `undefined` for a transport error. + * @returns `true` to retry once. + */ +export const isTransientProbeAnswer = (status: number | undefined): boolean => + status === undefined || status === 429 || status >= 500; + +/** + * Read one keyless answer. + * + * @param answer.status - the HTTP status. + * @param answer.bodyText - the response body, as text. + * @returns the verdict. + */ +export const classifyKeylessProbe = (answer: { + readonly status: number; + readonly bodyText: string; +}): Exclude => { + if (answer.status === 404) { + return "path-gone"; + } + if (answer.status === 410) { + return "model-gone"; + } + if (answer.status === 400) { + return /model is unavailable/iu.test(answer.bodyText) + ? "model-gone" + : "path-exists"; + } + if (answer.status === 401 || answer.status === 403) { + return "path-exists"; + } + return "other"; +}; diff --git a/test/support/src.ts b/test/support/src.ts index a8ea1fa2..b42ae157 100644 --- a/test/support/src.ts +++ b/test/support/src.ts @@ -168,6 +168,7 @@ export { isDeprecatedModelOn, isGoModelsListingUrl, isModelsListingUrl, + MODELS_OPENCODE_URL, parseModelsDevCatalog, refreshCatalog, resetCatalogState, @@ -209,6 +210,7 @@ export { export { BASE_FOR_PROTOCOL, baseForProtocol, + LEGACY_PLANE_API, piModelFor, planeBaseFor, protocolFor, diff --git a/vite.config.ts b/vite.config.ts index 5f2fff6e..c9ff8205 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -25,7 +25,12 @@ export default defineConfig({ }, overrides: [ { - files: ["test/**/*.ts", "test/**/*.tsx", "scripts/**/*.ts"], + files: [ + "test/**/*.ts", + "test/**/*.tsx", + "test/**/*.mjs", + "scripts/**/*.ts", + ], rules: { "class-methods-use-this": "off", "func-style": "off", From 409b5e12b4c99f184d349e2b10808043489130f1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 13:58:55 +0000 Subject: [PATCH 02/11] fix: address the second review of P-T1b The review ran three rounds through the correctness, rules and matrix lenses. What it found, and what changed: - The binary e2e's beforeAll budget was smaller than the blocking subprocess deadlines it serializes. Each plugin install now has its own deadline, the hook's budget is derived from the installs and the boot, and afterAll stops a host a failed hook left running. - providerWithApiKey claimed to be a line-for-line restatement but skipped upstream's credentialRef(ref) check. It now throws the same TypeError for a ref outside the grammar (a host-editor-contracts case pins it), and its comment names the deliberate differences. - The pinned e2e-host cell let cordis and the loader float at patch level. The overrides now force both, the cache key includes them, and the job asserts them against HOST_VERSIONS. - The pi-ai-first characterization asserted every facet under one KNOWN_DEFECT title. Each apply order now generates one test per row and facet, titled with that order's own tags. - CONTRIBUTING said the contract tests pin F11-F20, and the plan said P-T1 pins every fact. Both now give the same accounting: which facts are pinned by number, which P-R1 exercises first, and why F9 and F17 are not pinned. - CONTRIBUTING put the e2e's dummy keys in the profile's credentials file; they are in the temporary home's credentials document. - F5 had no real-host pin: host-shapes now pins that a route-level api and baseURL collapse pi's protocols onto one. F6 is recorded as first exercised by P-R1. - The credential-mapping row pinned only the api-key branch of toPiCredential. host-credentials now covers all three branches, and the env branch is asserted through pi's Models.getAuth, so the case fails without it. - The plan's reason for restating hasProviderApiKey was inaccurate; it now says no installed copy reaches the function. - The new credential cases cited toPiCredential at the wrong lines; they now cite auth.ts:74-84. - F4's listing and dispatch halves were pinned only by KNOWN_DEFECT rows P-R1 flips. A plugin-less tier-A case in host-shapes now pins that an id pi cannot place is not listed and that a turn on it is INVALID_CONFIG with no request; the save and deferred-read halves are tagged F4 too. - The one facet that depends on the apply order lost its tag when the pi-ai-first block was split. The fixture case now checks only structure (the facets whose value moves are the facets whose tag moves), and the exact pin is its own KNOWN_DEFECT case that boots no host. CONTRIBUTING and the plan say P-R1 drops todayRowsFor's override and turns that case into a BASELINE check in the same diff. Nothing under src/ changed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- .github/workflows/ci.yml | 38 ++++++-- CONTRIBUTING.md | 12 +-- docs/improvement-plan.md | 12 +-- test/e2e/host-binary.e2e.ts | 38 ++++++-- test/host-characterization.test.ts | 135 +++++++++++++++++++++-------- test/host-credentials.test.ts | 113 ++++++++++++++++++++++++ test/host-editor-contracts.test.ts | 40 ++++++++- test/host-shapes.test.ts | 75 +++++++++++++++- test/host-waterfall.test.ts | 2 +- test/support/host.ts | 53 ++++++++--- 10 files changed, 439 insertions(+), 79 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f063121..e2590ec0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -242,9 +242,11 @@ jobs: # a catalog-unknown id, and `test/e2e/host-binary.e2e.ts` asserts today's # rows (`S0_TODAY`, shared with the in-process characterization). # - # Two cells. `pinned` installs the host with npm `overrides` forcing pi to - # the version `test/support/host-versions.ts` names and applies the - # harness's own patch; `floating` takes whatever pi npm resolves for the + # Two cells. `pinned` installs the host with npm `overrides` forcing pi, + # cordis and the loader to the versions `test/support/host-versions.ts` + # names (the published host declares the last two as tilde ranges, so a + # fresh install could otherwise take a newer patch) and applies the + # harness's own patch to pi; `floating` takes whatever pi npm resolves for the # published host (newer than the harness pins), so a pi release that moves # the ground shows up nightly. Pull requests run `pinned` only, and only when # they touch the host seam; dispatch and the nightly run both cells. @@ -320,29 +322,33 @@ jobs: id: versions if: steps.scope.outputs.run == 'true' run: | - node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS } from './test/support/host-versions.ts'; console.log('dsh=' + HOST_VERSIONS.dsh); console.log('pi=' + HOST_VERSIONS.pi)" >> "$GITHUB_OUTPUT" + node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS } from './test/support/host-versions.ts'; console.log('dsh=' + HOST_VERSIONS.dsh); console.log('pi=' + HOST_VERSIONS.pi); console.log('cordis=' + HOST_VERSIONS.cordis); console.log('loader=' + HOST_VERSIONS.loader)" >> "$GITHUB_OUTPUT" echo "PREFIX=$RUNNER_TEMP/dsh-host" >> "$GITHUB_ENV" - # The pinned install is reproducible, so it is cached by the versions - # and the patch; the floating one is the point of re-resolving, so never. + # The pinned install forces every host version HOST_VERSIONS names, so it + # is cached by those versions and the patch (the rest of the published + # host's tree is whatever its ranges resolved on the miss that filled + # the cache); the floating one is the point of re-resolving, so never. - name: cache the pinned host install id: host-cache if: steps.scope.outputs.run == 'true' && matrix.cell == 'pinned' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/dsh-host - key: dsh-host-${{ matrix.cell }}-${{ steps.versions.outputs.dsh }}-${{ steps.versions.outputs.pi }}-${{ hashFiles('patches/*.patch') }} + key: dsh-host-${{ matrix.cell }}-${{ steps.versions.outputs.dsh }}-${{ steps.versions.outputs.pi }}-${{ steps.versions.outputs.cordis }}-${{ steps.versions.outputs.loader }}-${{ hashFiles('patches/*.patch') }} - name: install the published host into a throwaway prefix if: steps.scope.outputs.run == 'true' && steps.host-cache.outputs.cache-hit != 'true' env: CELL: ${{ matrix.cell }} DSH: ${{ steps.versions.outputs.dsh }} PI: ${{ steps.versions.outputs.pi }} + CORDIS: ${{ steps.versions.outputs.cordis }} + LOADER: ${{ steps.versions.outputs.loader }} run: | mkdir -p "$PREFIX" # The overrides must be in the prefix's own package.json BEFORE the # install, or npm resolves the wrapper's ^range to the newest pi. if [ "$CELL" = pinned ]; then - printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s" } }\n' "$PI" > "$PREFIX/package.json" + printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s", "@deepseek-ai/cordis": "%s", "@deepseek-ai/cordis-plugin-loader": "%s" } }\n' "$PI" "$CORDIS" "$LOADER" > "$PREFIX/package.json" else printf '{ "private": true }\n' > "$PREFIX/package.json" fi @@ -351,11 +357,13 @@ jobs: if [ "$CELL" = pinned ]; then patch -p1 -d "$PREFIX/node_modules/@earendil-works/pi-ai" < "patches/@earendil-works__pi-ai@$PI.patch" fi - - name: assert the pi the host resolved + - name: assert the pi, cordis and loader the host resolved if: steps.scope.outputs.run == 'true' env: CELL: ${{ matrix.cell }} PI: ${{ steps.versions.outputs.pi }} + CORDIS: ${{ steps.versions.outputs.cordis }} + LOADER: ${{ steps.versions.outputs.loader }} run: | resolved="$(node -p "require(process.env.PREFIX + '/node_modules/@earendil-works/pi-ai/package.json').version")" echo "cell $CELL: pi $resolved" @@ -368,6 +376,18 @@ jobs: echo "::error::the pinned cell resolved pi $resolved, not $PI" exit 1 fi + # The check test/host-versions.test.ts runs for the unit install, + # here for the host the binary e2e boots (a cached prefix included). + for pair in "cordis:$CORDIS" "cordis-plugin-loader:$LOADER"; do + name="@deepseek-ai/${pair%%:*}" + want="${pair#*:}" + got="$(node -p "require(process.env.PREFIX + '/node_modules/$name/package.json').version")" + echo "cell $CELL: $name $got" + if [ "$CELL" = pinned ] && [ "$got" != "$want" ]; then + echo "::error::the pinned cell resolved $name $got, not $want" + exit 1 + fi + done - if: steps.scope.outputs.run == 'true' run: pnpm run build - name: boot the host binary with the probe diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66529591..a474166e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ What the main files are for: ## The real-host harness -`test/support/host.ts` builds the host from the packages this repository installs, at the versions `test/support/host-versions.ts` names (pi 1.1.0 with the harness patch, the `@deepseek-ai` cluster at the published host's tag). The routing work is judged against it: the contract tests pin the host facts the design rests on (`docs/improvement-plan.md`, F11–F20), and the characterization rows record what today's plugin does. +`test/support/host.ts` builds the host from the packages this repository installs, at the versions `test/support/host-versions.ts` names (pi 1.1.0 with the harness patch, the `@deepseek-ai` cluster at the published host's tag). The routing work is judged against it: the contract tests pin the host facts the design rests on that the host can answer in-process (`docs/improvement-plan.md`: F1, F2, F4, F5, F8, F10–F16, F18 and F20, each by number; F6 and F19's auth half (constructing a `PiAiAdapter`) are first exercised by P-R1; F19's export half is pinned in `host-shapes` (`recordKeyFor`); F17 is honoured, not pinned — tier B fixes the apply order and the binary e2e infers it; F9 is a fact about opencode's own client, outside this host), and the characterization rows record what today's plugin does. Two tiers, both in-process: @@ -65,7 +65,7 @@ What every case does: - **Give tier-B cases 30 s**, and wait with a deadline (`waitForRoutes`, 10 s by default, so a route that never registers fails by name inside the hook's budget), never a fixed sleep: the plugin mounts its hidden routes asynchronously. - **A first row that fails does not hang the boot.** Tier B holds the second of pi-ai and the plugin until the first applies OR its fiber settles in any state, so a row whose config the schema rejects boots with that fiber FAILED (`fiber.await()` rethrows the real error) and the other row applied. -**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it) or `[INTENDED_CHANGE]` (right today; a phase changes it on purpose). A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY` in `host.ts`, which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, so the tags in `S0_TODAY` are what govern which entries a phase may flip. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. +**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it) or `[INTENDED_CHANGE]` (right today; a phase changes it on purpose). A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY` in `host.ts`, which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon-contributor`'s call config) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. **A catalog refresh can touch the rows.** `S0_TODAY`'s context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case compares each row's context window and effort ids to the bundled catalog's derivation and names the model that moved; when it fails on a refresh pull request, update the row in `S0_TODAY`. A phase that moves a non-`BASELINE` `info` facet off the catalog on purpose (the user's context window, a toggle model's Off/On) adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. @@ -73,17 +73,17 @@ What every case does: ### The `e2e-host` job -`.github/workflows/ci.yml`'s `e2e-host` job installs the published `@deepseek-ai/dsh` (at `HOST_VERSIONS.dsh`) into a throwaway prefix, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the profile's credentials file, boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. +`.github/workflows/ci.yml`'s `e2e-host` job installs the published `@deepseek-ai/dsh` (at `HOST_VERSIONS.dsh`) into a throwaway prefix, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the temporary home's credentials document (`$DSH_HOME/.credentials.yaml`, shared by every profile under that home), boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. -- **Cells.** `pinned` forces pi to `HOST_VERSIONS.pi` with npm `overrides` and applies the harness patch with `patch -p1`, then asserts every row. `floating` takes the pi npm resolves for the published host. It skips the rows that depend on which ids pi describes (the keyed listing and the per-model rows) and still asserts the save rows. Those assume the floating pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage a red floating run as a pi change first. Its stored-gap save row asserts only that SOME undescribed id blocks the save, not which one; its probe JSON is the record. +- **Cells.** `pinned` forces pi, cordis and the loader to `HOST_VERSIONS` with npm `overrides` (the published host declares cordis and the loader as tilde ranges), applies the harness patch to pi with `patch -p1`, asserts all three installed versions (a cached prefix included), then asserts every row. `floating` takes the pi npm resolves for the published host. It skips the rows that depend on which ids pi describes (the keyed listing and the per-model rows) and still asserts the save rows. Those assume the floating pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage a red floating run as a pi change first. Its stored-gap save row asserts only that SOME undescribed id blocks the save, not which one; its probe JSON is the record. - **Triggers.** Dispatch and the nightly run take both cells. A pull request takes `pinned`, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` or `patches/` (what the job installs and patches), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. To run it locally, install the host into a prefix outside the checkout, then point the suite at it (placeholder paths; `pnpm` must be on `PATH`). The first line reads the versions from `test/support/host-versions.ts`, the same way the CI job does, so the snippet never names one itself: ```sh -eval "$(node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS as v } from './test/support/host-versions.ts'; console.log('DSH=' + v.dsh + ' PI=' + v.pi)")" +eval "$(node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS as v } from './test/support/host-versions.ts'; console.log('DSH=' + v.dsh + ' PI=' + v.pi + ' CORDIS=' + v.cordis + ' LOADER=' + v.loader)")" mkdir -p /home/you/dsh-host -printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s" } }\n' "$PI" > /home/you/dsh-host/package.json +printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s", "@deepseek-ai/cordis": "%s", "@deepseek-ai/cordis-plugin-loader": "%s" } }\n' "$PI" "$CORDIS" "$LOADER" > /home/you/dsh-host/package.json npm install --prefix /home/you/dsh-host "@deepseek-ai/dsh@$DSH" patch -p1 -d /home/you/dsh-host/node_modules/@earendil-works/pi-ai < "patches/@earendil-works__pi-ai@$PI.patch" pnpm run build diff --git a/docs/improvement-plan.md b/docs/improvement-plan.md index 1ca3f82f..b323e4a8 100644 --- a/docs/improvement-plan.md +++ b/docs/improvement-plan.md @@ -123,7 +123,7 @@ Phases land as **stacked pull requests**. Each one is based on the previous phas **The problem, in the owner's words.** A provider defines one backend. Supporting every protocol and the newer catalog meant un-registering the original `opencode`/`opencode-go` provider and registering the plugin's own. Deleting another plugin's registration is unsupported, so the plugin mounted extra routes per protocol instead, merged their models back into the user's provider, and redirected dispatch. This section shows that the host **does** allow the original intent, through supported paths (F11–F13): keep pi-ai from claiming the two ids, and register the plugin's provider under them. A spike on the real host confirmed it (run S0). -**Facts from the host's own code and data.** F1–F5 and F10 are measured: the data files were counted and the resolver read. F6–F9 and F11–F13 are read from the code, and run S0 exercised F11–F13 end to end. P-T1's real-host harness pins them all, and a fact that fails there sends the design back to this section. +**Facts from the host's own code and data.** F1–F5 and F10 are measured: the data files were counted and the resolver read. F6–F9 and F11–F13 are read from the code, and run S0 exercised F11–F13 end to end. P-T1's real-host harness pins the ones the host can answer in-process: P-T1b pins F1, F2, F4, F5, F8, F10–F16, F18 and F20 by number (F3 through its F4 consequence, F7 through F13's register and DUPLICATE halves); F6 (dispatch through a non-pi adapter) and F19's auth half (constructing a `PiAiAdapter`) are first exercised by P-R1, F6 through its mock gateway and the real `LlmRuntime`; F19's export half is pinned in `host-shapes` (`recordKeyFor`); F17 is honoured, not pinned — tier B fixes the apply order and the binary e2e infers it; F9 is a fact about opencode's own client, outside this host. A fact that fails where it is pinned sends the design back to this section. | # | Fact | Evidence | | :-- | :-- | :-- | @@ -325,7 +325,7 @@ What stays private is pinned by contract tests against the real host packages: - `BASELINE`: right today and must survive every phase. No phase may move a `BASELINE` facet, except that P-R2's measured family flip, if it happens, may move the endpoint-family part of a wire facet; it re-tags each one it moves and names it in its PR body. - `KNOWN_DEFECT`, wrong today and a phase fixes it, for example: - - Go Gemini answers `INVALID_CONFIG` — in the published host's apply order (`plugin-first`); with pi-ai applying first it validates, so the row is order-dependent (`todayRowsFor`); + - Go Gemini answers `INVALID_CONFIG` — in the published host's apply order (`plugin-first`); with pi-ai applying first it validates, so the row is order-dependent (`todayRowsFor`). P-R1 flips it and, in the same diff, drops `todayRowsFor`'s pi-ai-first override and turns the `[KNOWN_DEFECT]` apply-order case into a `[BASELINE]` "no facet depends on the apply order" check; - an unknown id is rejected; - toggle models show Minimal/Xhigh/Max. - `INTENDED_CHANGE`, right today and a phase changes it on purpose, for example: @@ -360,7 +360,7 @@ What stays private is pinned by contract tests against the real host packages: - Rewrite `mount-protocols.test.ts` as "every catalog SDK maps to a protocol that pi's `api/*` exports implement", keeping the row that pi 1.1.0's Go provider lacks Google. 5. **P-T1b `test:` the real-host harness.** Additive only; in review. It landed `test/support/host.ts` (tier A: `Context` + `LlmRuntime` + pi-ai; tier B: app-boot `boot()` with a temporary HOME), the contract tests (`host-waterfall`, `host-shapes`, `host-credentials`, `host-editor-contracts`), the tagged characterization (`host-characterization`, with `S0_TODAY` shared by the binary e2e), the keyless matrix, the probe bundle and the `e2e-host` job. Two departures from the phase as first planned, both deliberate: tier B's rows carry package names through a stub module loader rather than `cordis:` builtins, because the plugin finds pi-ai by `options.name` (`piAiEntry` in `src/responses-provider.ts`); and tier B fixes the apply order (`plugin-first`, the binary's, or `pi-ai-first`), because the loader starts rows in parallel. Two narrower ones: the `e2e-host` PR scope is wider than first named (the job's own inputs, and until P-R1 the `src` modules that install the host patches and mounts whose behaviour depends on the real loader and apply order), and the floating cell skips the rows that depend on which ids pi describes. - The `hasProviderApiKey` baseline (`api/session-controller/src/catalog.ts:97-116`) is a **restatement**: session-controller does not export the function (its `exports` map reaches `SessionController` and `buildModelCatalog` only), so `test/support/host.ts` rebuilds it line for line (`providerWithApiKey`) from the real parts tier B holds — `settings.describe({ redactSecrets: true })`, `llm.listConfigurableProviders()` and `credentials.describe` — and both `host-editor-contracts` and `host-characterization` use it. The cases in `host-editor-contracts` boot WITHOUT the plugin, so they are host contracts that nothing in `src/` can move: with only the plane keys stored it answers true (the Zen plane); under a test-local strip listener it answers false, because the planes' `describe().value` is gone (F20), which upstream makes `initializeDefaultModel` replace the user's default model. That false is the host fact the D18 wrapper exists to counter, not a row any phase flips. The guard on the plugin is the `[BASELINE]` row in `host-characterization`: today's plugin as it ships, with only the plane keys stored, in both apply orders, answers the Zen plane, and pi-ai's `settings.describe` value lists both planes. P-R1 must keep that row green while its own strip is active. + The `hasProviderApiKey` baseline (`api/session-controller/src/catalog.ts:97-116`) is a **restatement**: no installed copy reaches the function (the published `@deepseek-ai/dsh-api-session-controller` entry exports `SessionController`, `buildModelCatalog` and a few classes but not it, its tarball ships `lib/` and no `src/`, and the package is not a dependency here in any case), so `test/support/host.ts` restates it (`providerWithApiKey`) from the real parts tier B holds — `settings.describe({ redactSecrets: true })`, `llm.listConfigurableProviders()` and `credentials.describe` — keeping upstream's `credentialRef` grammar check as a `TypeError` (a host-editor-contracts case pins it), answering the provider id where upstream answers `true`, and omitting the `RemoteError` for a missing service tier B always has; both `host-editor-contracts` and `host-characterization` use it. The cases in `host-editor-contracts` boot WITHOUT the plugin, so they are host contracts that nothing in `src/` can move: with only the plane keys stored it answers true (the Zen plane); under a test-local strip listener it answers false, because the planes' `describe().value` is gone (F20), which upstream makes `initializeDefaultModel` replace the user's default model. That false is the host fact the D18 wrapper exists to counter, not a row any phase flips. The guard on the plugin is the `[BASELINE]` row in `host-characterization`: today's plugin as it ships, with only the plane keys stored, in both apply orders, answers the Zen plane, and pi-ai's `settings.describe` value lists both planes. P-R1 must keep that row green while its own strip is active. **What remains:** the Models store's join. It lives in the client package `ui-settings-models`, which is not installed here and has no server-side seam to restate, so P-R1 owns it (its contract list below, unconditionally), measured on the binary or with the package added at the pinned cluster tag. @@ -471,7 +471,7 @@ P0 → P1a → P1b → P2 → P-T1a → P-T1b → P-H → P-R1 → P-R2 → P3 - the listener fires for another plugin's fiber, and `prepend` puts it outermost; - `raw === _config` holds for `_reload`, `update` and `_commitVolatile`, and not for the editor's candidate (F14); - a deep-frozen input is never mutated, and the fiber keeps the return value (F12); - - the profile shape, the `Config` defaults, the credential record mapping and `recordKeyFor`; + - the profile shape, the `Config` defaults, the credential record mapping (each of `toPiCredential`'s three branches: an api-key record with a key, one carrying `env` too — observable on a plane only in pi's `Models.getAuth` answer, since no plane request reads it — and a non-api-key payload handed through) and `recordKeyFor`; - no secret-role nodes; - the loader's `interpolate`; - pi's public exports. @@ -565,8 +565,8 @@ P0 → P1a → P1b → P2 → P-T1a → P-T1b → P-H → P-R1 → P-R2 → P3 **What it does:** -1. Installs `@deepseek-ai/dsh@$DSH_VERSION` into a throwaway prefix, cached by host version, with npm `overrides` forcing pi. -2. Asserts the pi version that resolved. +1. Installs `@deepseek-ai/dsh@$DSH_VERSION` into a throwaway prefix, cached by host version, with npm `overrides` forcing pi (and, in the pinned cell, cordis and the loader, which the published host declares as tilde ranges). +2. Asserts the pi version that resolved, and in the pinned cell the cordis and loader versions against `HOST_VERSIONS`. 3. Builds, and boots a temporary HOME whose profile links the checkout through a relative `link:`. 4. Runs `test/e2e/host/probe-plugin.mjs` together with `test/e2e/host-binary.e2e.ts`, which runs only when `DSH_HOST_E2E=1`. diff --git a/test/e2e/host-binary.e2e.ts b/test/e2e/host-binary.e2e.ts index 76fcf5e2..e88041d8 100644 --- a/test/e2e/host-binary.e2e.ts +++ b/test/e2e/host-binary.e2e.ts @@ -39,7 +39,7 @@ * * @module test/e2e/host-binary.e2e */ -import { spawn, spawnSync } from "node:child_process"; +import { type ChildProcess, spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, @@ -74,6 +74,16 @@ const PREFIX = process.env.DSH_HOST_PREFIX ?? ""; const CELL = process.env.DSH_HOST_PI_CELL ?? "pinned"; const PINNED = CELL === "pinned"; const BOOT_DEADLINE_MS = 180_000; +/** Each `dsh plugin install` (a blocking `spawnSync`). */ +const INSTALL_DEADLINE_MS = 120_000; +/** + * The hook's budget, derived from the deadlines it actually runs — two installs + * and a boot — so every failure path reaches its own message and the probe + * copy before vitest's hook timeout can fire. `spawnSync` blocks the worker, so + * the hook timer cannot interrupt an install; it can only fire during the boot + * wait, which is the window where the child is live and the artifact matters. + */ +const HOOK_BUDGET_MS = 2 * INSTALL_DEADLINE_MS + BOOT_DEADLINE_MS + 30_000; const CHECKOUT = fileURLToPath(new URL("../..", import.meta.url)); const PROBE_BUNDLE = join(CHECKOUT, "test", "e2e", "host"); @@ -202,6 +212,8 @@ const posix = (path: string): string => path.split(sep).join("/"); let home = ""; let out: ProbeOutput; +/** The booted host, kept so `afterAll` can stop it before removing HOME. */ +let child: ChildProcess | undefined; describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { beforeAll(async () => { @@ -255,7 +267,7 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { cwd: home, encoding: "utf8", env, - timeout: BOOT_DEADLINE_MS, + timeout: INSTALL_DEADLINE_MS, }); if (run.status !== 0) { // A spawn error, a timeout and a signal all leave `status` null; name @@ -315,7 +327,7 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { ); const probeOut = join(home, "probe.json"); - const child = spawn( + const host = spawn( process.execPath, [bin, "--profile", "e2e", "--no-open", "--port", "0"], { @@ -332,17 +344,18 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { stdio: ["ignore", "pipe", "pipe"], } ); + child = host; let stderr = ""; - child.stderr.on("data", (chunk: Buffer) => { + host.stderr.on("data", (chunk: Buffer) => { stderr = `${stderr}${chunk.toString("utf8")}`.slice(-4000); }); - child.stdout.resume(); + host.stdout.resume(); const end = await new Promise((resolve) => { const timer = setTimeout(() => { - child.kill("SIGKILL"); + host.kill("SIGKILL"); resolve({ code: null, signal: "SIGKILL", timedOut: true }); }, BOOT_DEADLINE_MS); - child.on("exit", (code, signal) => { + host.on("exit", (code, signal) => { clearTimeout(timer); resolve({ code, signal, timedOut: false }); }); @@ -374,9 +387,18 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { } out = parsed; console.info(`[e2e-host] probe output:\n${JSON.stringify(out, null, 2)}`); - }, BOOT_DEADLINE_MS * 2); + }, HOOK_BUDGET_MS); afterAll(() => { + // A hook that failed before the boot settled leaves the host running; + // stop it before its HOME goes. + if ( + child !== undefined && + child.exitCode === null && + child.signalCode === null + ) { + child.kill("SIGKILL"); + } if (home.length > 0) { rmSync(home, { force: true, recursive: true }); } diff --git a/test/host-characterization.test.ts b/test/host-characterization.test.ts index 70d96a3c..1125f822 100644 --- a/test/host-characterization.test.ts +++ b/test/host-characterization.test.ts @@ -16,8 +16,9 @@ * the binary e2e reads too, so the in-process host and the published binary * cannot drift apart unnoticed. Each model's three facets (model info, the * call-config verdict, the wire) carry their own tag there, and this file - * generates one test per model and facet titled with that tag, so the tag is - * what a reader and a phase's diff go by. Where a measurement contradicted the + * generates one test per model and facet titled with that tag, in each apply + * order (`todayRowsFor(order)` carries that order's tags), so the tag is what + * a reader and a phase's diff go by. Where a measurement contradicted the * plan's expectation the row keeps the measurement and says so. * * Every case boots its own host, so a key the fetch patch captured in one case @@ -230,14 +231,21 @@ const dropFrom = }; }; -/** One generated case per row and facet, titled with the facet's own tag. */ -const FACET_CASES = ROW_KEYS.flatMap((key) => - (["info", "callConfig", "wire"] as const).map((facet) => ({ - facet, - key, - tag: S0_TODAY.rows[key].tags[facet], - })) -); +const FACETS = ["info", "callConfig", "wire"] as const; + +/** + * One generated case per row and facet, titled with the facet's own tag in + * that order's table, so a phase's diff names every facet it moves in both + * orders. + */ +const facetCasesFor = (order: ApplyOrder) => + ROW_KEYS.flatMap((key) => + FACETS.map((facet) => ({ + facet, + key, + tag: todayRowsFor(order)[key].tags[facet], + })) + ); /** The plane a row key belongs to, and the bundled catalog for it. */ const catalogFor = (key: RowKey) => @@ -377,7 +385,7 @@ describe( expect(compared).toBeGreaterThan(0); }); - it("fixture: the binary e2e's cell-aware helpers — the floating save-block pattern names any undescribed id and only those, and todayRowsFor refuses an order it does not know", () => { + it("fixture: the binary e2e's cell-aware helpers — the floating save-block pattern names any undescribed id and only those, todayRowsFor refuses an order it does not know, and a facet its pi-ai-first table moves is retagged with it, from KNOWN_DEFECT to BASELINE", () => { // The pinned message is one the floating pattern accepts, so the two // cells cannot disagree on a pinned run. const pinned = @@ -397,6 +405,39 @@ describe( for (const order of APPLY_ORDERS) { expect(Object.keys(todayRowsFor(order))).toEqual(ROW_KEYS); } + // Structural, so no phase has to edit it: a facet whose VALUE differs + // between the orders is exactly a facet whose TAG differs, and each one + // is a defect in the published order whose pi-ai-first value is already + // what P-R1 keeps. Which facets those are today is pinned by the + // [KNOWN_DEFECT] apply-order case below; once P-R1 collapses the + // override, both sets are empty and this still holds. + const piAiFirst = todayRowsFor("pi-ai-first"); + const moved = ROW_KEYS.flatMap((key) => + FACETS.filter( + (facet) => + JSON.stringify(expectedFacet(facet, key, piAiFirst)) !== + JSON.stringify(expectedFacet(facet, key)) + ).map((facet) => `${key} · ${facet}`) + ); + const retagged = ROW_KEYS.flatMap((key) => + FACETS.filter( + (facet) => + piAiFirst[key].tags[facet] !== S0_TODAY.rows[key].tags[facet] + ).map((facet) => `${key} · ${facet}`) + ); + expect(retagged).toEqual(moved); + for (const key of ROW_KEYS) { + for (const facet of FACETS) { + if (moved.includes(`${key} · ${facet}`)) { + expect(S0_TODAY.rows[key].tags[facet], `${key} · ${facet}`).toBe( + "KNOWN_DEFECT" + ); + expect(piAiFirst[key].tags[facet], `${key} · ${facet}`).toBe( + "BASELINE" + ); + } + } + } // A probe JSON missing the field reads `undefined`; it must not select // the pi-ai-first table. expect(() => todayRowsFor(undefined as unknown as ApplyOrder)).toThrow( @@ -476,9 +517,14 @@ describe( } }); - it.each(FACET_CASES)("[$tag] $key · $facet", async ({ facet, key }) => { - expect(await measureFacet(facet, key)).toEqual(expectedFacet(facet, key)); - }); + it.each(facetCasesFor("plugin-first"))( + "[$tag] $key · $facet", + async ({ facet, key }) => { + expect(await measureFacet(facet, key)).toEqual( + expectedFacet(facet, key) + ); + } + ); it("[KNOWN_DEFECT] the toggle-only model is offered Minimal/Xhigh/Max (D1)", async () => { // Chosen from the bundled catalog, not by name: on/off reasoning, no levels. @@ -655,31 +701,50 @@ describe( } }); - it("[KNOWN_DEFECT] the Go Gemini verdict depends on the apply order: decorated, gemini-4-argon-contributor validates, and no other facet of any row moves (decoratePlaneAdapters in src/models-discovery.ts)", async () => { - // The plan expected INVALID_CONFIG for Go Gemini; that holds in the - // published host's order (above) and not in this one, where the wrapped - // plane adapter delegates to opencode-go-google. todayRowsFor is the - // table the binary e2e picks when its probe infers this order. - const expected = todayRowsFor("pi-ai-first"); - expect( - ROW_KEYS.filter( - (key) => - JSON.stringify(expected[key]) !== JSON.stringify(S0_TODAY.rows[key]) - ) - ).toEqual(["opencode-go/gemini-4-argon-contributor"]); - for (const facet of ["info", "callConfig", "wire"] as const) { - const measured: Record = {}; - const want: Record = {}; - for (const key of ROW_KEYS) { - measured[key] = await measureFacet(facet, key); - want[key] = expectedFacet(facet, key, expected); - } - expect(measured, facet).toEqual(want); + // The plan expected INVALID_CONFIG for Go Gemini; that holds in the + // published host's order (the plugin-first block, where the case is + // [KNOWN_DEFECT]) and not in this one, where the wrapped plane adapter + // delegates to opencode-go-google (decoratePlaneAdapters in + // src/models-discovery.ts), so here it is [BASELINE]. todayRowsFor is the + // table the binary e2e picks when its probe infers this order; these cases + // measure every facet of it, and the apply-order case below pins that + // only that one facet moves. + it.each(facetCasesFor("pi-ai-first"))( + "[$tag] $key · $facet", + async ({ facet, key }) => { + expect(await measureFacet(facet, key)).toEqual( + expectedFacet(facet, key, todayRowsFor("pi-ai-first")) + ); } - }); + ); } ); +describe("characterization · the apply order (no host)", () => { + // Pure: it reads the two tables the binary e2e picks between, and boots + // nothing. Each order's generated facet cases measure its table; this case + // pins WHERE they differ today. The dependence on the order is the defect: + // when P-R1 flips this row it drops todayRowsFor's pi-ai-first override in + // the same diff, and this case becomes "[BASELINE] no facet depends on the + // apply order" (CONTRIBUTING.md, "Tags"). + it("[KNOWN_DEFECT] the apply order decides exactly one facet today: Go gemini-4-argon-contributor's callConfig is INVALID_CONFIG in the published order and \"ok\" with pi-ai first (P-R1 drops todayRowsFor's pi-ai-first override with this row's flip)", () => { + const piAiFirst = todayRowsFor("pi-ai-first"); + const moved = ROW_KEYS.flatMap((key) => + FACETS.filter( + (facet) => + JSON.stringify(expectedFacet(facet, key, piAiFirst)) !== + JSON.stringify(expectedFacet(facet, key)) + ).map((facet) => `${key} · ${facet}`) + ); + expect(moved).toEqual([ + "opencode-go/gemini-4-argon-contributor · callConfig", + ]); + const key = "opencode-go/gemini-4-argon-contributor"; + expect(expectedFacet("callConfig", key)).toBe("INVALID_CONFIG"); + expect(expectedFacet("callConfig", key, piAiFirst)).toBe("ok"); + }); +}); + describe( "characterization · a gap-free plane (tier B)", { timeout: 30_000 }, diff --git a/test/host-credentials.test.ts b/test/host-credentials.test.ts index 8e0377a1..504b8ab4 100644 --- a/test/host-credentials.test.ts +++ b/test/host-credentials.test.ts @@ -16,6 +16,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { + adaptersOf, drainStream, fakeCredentials, fakeLaunchEnvironment, @@ -32,6 +33,38 @@ afterEach(async () => { vi.unstubAllGlobals(); }); +/** What pi's `Models.getAuth` answers, as far as these cases read it. */ +interface PiAuthResult { + readonly env?: Readonly>; + readonly source?: string; +} + +/** + * pi's own auth answer for a route, through the collection pi-ai's adapter + * built for its current snapshot (adapter.ts:251-259, private) and pi's public + * `Models.getAuth` (pi 1.1.0 `dist/models.d.ts`). It is the one place a stored + * record's `env` is observable on a plane: pi forwards it as the result's + * `env` (`dist/auth/helpers.js`), and no plane request reads it back. + * + * @param current - the tier whose pi-ai adapter to read. + * @param route - the route to resolve auth for. + * @returns the source and env, with the key left out. + */ +const piAuthOf = async ( + current: TierA, + route: string +): Promise => { + const adapter = adaptersOf(current.llm).get(route)?.adapter as unknown as { + current: () => { + models: { getAuth: (id: string) => Promise }; + }; + }; + const result = await adapter.current().models.getAuth(route); + return result === undefined + ? undefined + : { env: result.env, source: result.source }; +}; + /** One completions model on Zen, which pi 1.1.0 serves at its legacy base. */ const ZEN_COMPLETIONS = { models: [{ id: "deepseek-v4-pro" }] }; const LEGACY_COMPLETIONS = "https://opencode.ai/zen/v1/chat/completions"; @@ -59,6 +92,86 @@ describe("host · pi-ai's credential paths (tier A)", () => { "llm-pi-ai/opencode", ]); expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); + // A record with no env hands pi none (auth.ts:80 spreads it only when set). + expect(await piAuthOf(tier, "opencode")).toEqual({ + env: undefined, + source: "stored credential", + }); + }); + + it("maps an api-key record carrying env as well: pi's auth carries the stored env, the turn still authenticates with the stored key, and the launch environment is not read (auth.ts:74-84)", async () => { + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const credentials = fakeCredentials({ + records: { + "llm-pi-ai/opencode": { + env: { OPENCODE_REGION: "test" }, + key: "stored-k", + kind: "api-key", + }, + }, + }); + const launchEnvironment = fakeLaunchEnvironment({ + OPENCODE_API_KEY: "env-key-k", + }); + tier = await mountTierA({ + credentials, + launchEnvironment, + providers: { opencode: ZEN_COMPLETIONS }, + }); + const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(wire.requests.map((request) => request.url)).toEqual([ + LEGACY_COMPLETIONS, + ]); + // The stored key's length, not the environment's (a different length). + expect(wire.requests[0]?.headers.authorization).toBe( + "Bearer stored-k".length + ); + expect(launchEnvironment.reads).toEqual([]); + expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); + // The env branch itself (auth.ts:80): nothing on a plane's wire reads the + // stored env, so it is asserted where pi hands it on, in its auth answer. + expect(await piAuthOf(tier, "opencode")).toEqual({ + env: { OPENCODE_REGION: "test" }, + source: "stored credential", + }); + }); + + it("hands a non-api-key record's payload to pi verbatim: on a plane, whose pi auth has no OAuth, the turn fails PI_AI_ERROR with no request and no fallback to the launch environment (auth.ts:74-84)", async () => { + // Not a record a plane's Sign-in writes (pi 1.1.0's planes offer an API + // key only); pinned so P-R1's store, which copies this mapping, answers + // the same for one a user or another plane left behind. + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const credentials = fakeCredentials({ + records: { + "llm-pi-ai/opencode": { + kind: "grant", + payload: { + access: "grant-access", + expires: Date.now() + 3_600_000, + refresh: "grant-refresh", + type: "oauth", + }, + }, + }, + }); + const launchEnvironment = fakeLaunchEnvironment({ + OPENCODE_API_KEY: "env-k", + }); + tier = await mountTierA({ + credentials, + launchEnvironment, + providers: { opencode: ZEN_COMPLETIONS }, + }); + const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); + expect(credentials.calls).toContainEqual([ + "readRecord", + "llm-pi-ai/opencode", + ]); + expect(wire.requests).toEqual([]); + expect(launchEnvironment.reads).toEqual([]); + expect(drained.finish).toEqual({ code: "PI_AI_ERROR", kind: "error" }); }); it("F18 — a named ref the credentials service misses is MISSING_CREDENTIAL with no request, and listModels answers [] (index.ts:188-211; adapter.ts:295-312)", async () => { diff --git a/test/host-editor-contracts.test.ts b/test/host-editor-contracts.test.ts index 1c9bd1e7..3c042fd1 100644 --- a/test/host-editor-contracts.test.ts +++ b/test/host-editor-contracts.test.ts @@ -228,7 +228,7 @@ describe( ]); }); - it("the stripping listener does not shield pi-ai's own save check: with a gap id stored, a plane edit is refused and the file is untouched (llm-pi-ai/src/index.ts:177-186; config.ts:364-368)", async () => { + it("F4 — the stripping listener does not shield pi-ai's own save check: with a gap id stored, a plane edit is refused and the file is untouched (llm-pi-ai/src/index.ts:177-186; config.ts:364-368)", async () => { // Read for P-R1. An outermost listener's copy is what the fiber KEEPS, // but pi-ai validates the value its inner chain returns — the editor's // raw candidate — and against its stripped previous config every plane @@ -317,6 +317,44 @@ describe( expect(await providerWithApiKey(tier)).toBe("deepseek"); }); + it("restatement — an apiKeyEnv outside credentialRef's grammar throws upstream's TypeError instead of reading 'not configured' (credentials/src/index.ts:29-34)", async () => { + // pi-ai's resolveProfiles refuses such a ref today, so only a describe + // graft (P-R1's) could carry one; the restatement must then fail the + // way upstream's initializeDefaultModel would, not skip the provider. + tier = await bootTierB({ + credentials: planeKeysOnly(), + plugin: false, + providers: s0Providers(), + }); + const { settings } = tier.ctx; + const malformedDescribe: typeof settings.describe = (options) => + settings.describe(options).map((namespace) => { + if (namespace.ns !== "llm-pi-ai") { + return namespace; + } + const value = structuredClone(namespace.value) as { + providers: { opencode: Record }; + }; + value.providers.opencode.apiKeyEnv = "OPENCODE API KEY"; + return { ...namespace, value }; + }); + const view = { + credentials: tier.credentials, + ctx: { settings: { describe: malformedDescribe } }, + llm: tier.llm, + } as unknown as TierB; + const from = tier.credentials.calls.length; + await expect(providerWithApiKey(view)).rejects.toThrow( + new TypeError( + 'credential ref "OPENCODE API KEY" must match /^[A-Za-z_][A-Za-z0-9_]*$/' + ) + ); + // The malformed ref is never handed to the credentials service. + expect(describedRefs(tier, from)).toEqual(["DEEPSEEK_API_KEY"]); + // Control: the unmodified view answers the plane, so the throw is the ref's. + expect(await providerWithApiKey(tier)).toBe("opencode"); + }); + it("host contract, F20 — under a test-local strip it answers FALSE with the plane keys stored: the directory still lists the planes, but their describe().value is gone, so their keys are never asked about", async () => { // Why P-R1 needs the settings.describe wrapper (D18): upstream, false // here makes initializeDefaultModel replace the user's default with the diff --git a/test/host-shapes.test.ts b/test/host-shapes.test.ts index 30ee2be1..9939b43f 100644 --- a/test/host-shapes.test.ts +++ b/test/host-shapes.test.ts @@ -21,11 +21,14 @@ import { interpolate } from "@deepseek-ai/cordis-plugin-loader"; import { resolveRetryPolicy } from "@deepseek-ai/dsh-llm"; import * as LlmPiAi from "@deepseek-ai/dsh-llm-pi-ai"; import z from "@deepseek-ai/schemastery"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { adaptersOf, + drainStream, + fakeCredentials, mountTierA, + recordingFetch, s0Providers, type TierA, } from "./support/host.ts"; @@ -143,6 +146,7 @@ let tier: TierA | undefined; afterEach(async () => { await tier?.dispose(); tier = undefined; + vi.unstubAllGlobals(); }); describe("host · pi-ai's Config (llm-pi-ai/src/config.ts)", () => { @@ -260,7 +264,7 @@ describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,4 ]); }); - it("resolves a profile with these keys at runtime, models destructured out", async () => { + it("resolves a profile with these keys at runtime, models destructured out; F4 — a stored id pi cannot place is deferred to modelErrors and catalogError (config.ts:487)", async () => { tier = await mountTierA({ providers: s0Providers() }); const profile = profileOf(tier, "opencode"); expect(Object.keys(profile).toSorted()).toEqual([ @@ -300,6 +304,48 @@ describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,4 ]).toEqual(["gemini-4-argon-contributor", "minimax-m3.1-flash"]); }); + it("F4 — a stored id pi cannot place is not listed, and a turn on it is INVALID_CONFIG with no request (index.ts:171; adapter.ts:271-311)", async () => { + // Plugin-less, so this half of F4 outlives the characterization rows that + // also show it with the plugin mounted (P-R1 flips those). A key is stored + // for each plane: with none, F18's empty listing would mask the check. + const wire = recordingFetch(); + vi.stubGlobal("fetch", wire.fetch); + const mounted = await mountTierA({ + credentials: fakeCredentials({ + refs: { OPENCODE_API_KEY: "zen-k", OPENCODE_GO_API_KEY: "go-k" }, + }), + providers: s0Providers(), + }); + tier = mounted; + const listed = async (route: string): Promise => { + const models = await mounted.llm.listModels(route); + return models.map((model) => model.id); + }; + expect(await listed("opencode")).toEqual([ + "deepseek-v4-pro", + "gpt-5.4", + "claude-haiku-5-5", + "gemini-3-flash", + ]); + expect(await listed("opencode-go")).toEqual([ + "deepseek-v4-pro", + "muse-spark-1.2-contributor", + "minimax-m3", + ]); + for (const [route, model] of [ + ["opencode", "gpt-5.3-codex-spark"], + ["opencode-go", "gemini-4-argon-contributor"], + ["opencode-go", "minimax-m3.1-flash"], + ] as const) { + const drained = await drainStream(mounted.llm, route, model); + expect(drained.finish, `${route}/${model}`).toEqual({ + code: "INVALID_CONFIG", + kind: "error", + }); + } + expect(wire.requests).toEqual([]); + }); + it("F1/F2 — a model-less plane resolves pi 1.1.0's whole catalog, per protocol, on the legacy bases", async () => { tier = await mountTierA({ providers: { opencode: {}, "opencode-go": {} }, @@ -331,6 +377,31 @@ describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,4 }, }); }); + + it("F5 — a route-level api and baseURL override every model, collapsing pi's four protocols onto one (llm-pi-ai/src/catalog.ts:917,922)", async () => { + tier = await mountTierA({ + providers: { + opencode: { api: "openai-responses" }, + "opencode-go": { baseURL: "https://gateway.test/go" }, + }, + }); + const zen = profileOf(tier, "opencode"); + expect(zen.catalogError).toBeUndefined(); + // Every one of F2's 83 Zen models now speaks Responses; each keeps its own + // base, which only the route's baseURL would override. + expect(byProtocol(zen.piProvider)).toEqual({ + "openai-responses": { + bases: ["https://opencode.ai/zen", "https://opencode.ai/zen/v1"], + count: 83, + }, + }); + // Go's 30 models keep their protocols and all move to the route's base. + expect(byProtocol(profileOf(tier, "opencode-go").piProvider)).toEqual({ + "anthropic-messages": { bases: ["https://gateway.test/go"], count: 5 }, + "openai-completions": { bases: ["https://gateway.test/go"], count: 19 }, + "openai-responses": { bases: ["https://gateway.test/go"], count: 6 }, + }); + }); }); describe("host · pi-ai's credential key (llm-pi-ai/src/auth.ts:29-39; credentials/src/index.ts:69-78)", () => { diff --git a/test/host-waterfall.test.ts b/test/host-waterfall.test.ts index 8e1f45a6..c2440720 100644 --- a/test/host-waterfall.test.ts +++ b/test/host-waterfall.test.ts @@ -151,7 +151,7 @@ describe("host · the internal/config waterfall (tier A)", () => { expect(tier.piFiber.state).toBe(2); }); - it("F8 — pi-ai's own listener validates the value its inner chain returns, not what an outer listener will return (llm-pi-ai/src/index.ts:177-186; config.ts:364-368)", async () => { + it("F8/F4 — pi-ai's own listener validates the value its inner chain returns, not what an outer listener will return (llm-pi-ai/src/index.ts:177-186; config.ts:364-368)", async () => { // Read for P-R1: an outer listener that strips the planes and returns a // copy does NOT shield pi-ai's save check, which runs on `next()`'s value. // A listener INSIDE pi-ai's (registered after it, unprepended) is what diff --git a/test/support/host.ts b/test/support/host.ts index 9f486934..ca125a74 100644 --- a/test/support/host.ts +++ b/test/support/host.ts @@ -706,7 +706,10 @@ export const ZEN_UNDESCRIBED = ["gpt-5.3-codex-spark", UNKNOWN_ID] as const; * TODAY's rows for an apply order. `S0_TODAY.rows` is the `plugin-first` * table; with `pi-ai-first` the adapter decoration wraps pi-ai's plane adapter * and exactly one facet moves — Go gemini-4-argon-contributor validates — - * measured in `test/host-characterization.test.ts`. The binary e2e picks the + * measured in `test/host-characterization.test.ts`, which generates one test + * per row and facet for each order, titled with that order's tags. The moved + * facet is `BASELINE` in this table: its `"ok"` is already what P-R1 keeps, so + * there is nothing for a phase to flip in this order. The binary e2e picks the * table from the order its probe INFERRED — whether the decoration wrapped * pi-ai's plane adapter, which is what the rows depend on — because on a real * host the order is a race (F17) the suite must not pin. @@ -730,6 +733,10 @@ export const todayRowsFor = ( "opencode-go/gemini-4-argon-contributor": { ...S0_TODAY.rows["opencode-go/gemini-4-argon-contributor"], callConfig: "ok", + tags: { + ...S0_TODAY.rows["opencode-go/gemini-4-argon-contributor"].tags, + callConfig: B, + }, }, }; }; @@ -1367,22 +1374,41 @@ export const planeKeysOnly = (): FakeCredentials => refs: { OPENCODE_API_KEY: "zen-test", OPENCODE_GO_API_KEY: "go-test" }, }); +/** + * `credentialRef`'s grammar (`credentials/credentials/src/index.ts:19,29-34`): + * a POSIX shell identifier. Restated because `@deepseek-ai/dsh-credentials` is + * only a transitive dependency here. + */ +const CREDENTIAL_REF_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + /** * session-controller's `hasProviderApiKey` (`api/session-controller/src/catalog.ts:97-116`), - * restated line for line from the real parts, because the package does not - * export it (its `exports` map reaches `SessionController` and - * `buildModelCatalog` only). `initializeDefaultModel` (`index.ts:320-331`) - * calls it after a Sign-in and, when it answers false, replaces the user's - * default model with the account's first one. It answers the FIRST - * configurable provider whose profile, read from `settings.describe`'s VALUE - * at the provider's `settingsPath`, names an `apiKeyEnv` the credentials - * service reports as configured — so it is the reader F20 changes. + * restated from the real parts because no installed copy reaches it: the + * published `@deepseek-ai/dsh-api-session-controller` entry exports + * `SessionController`, `buildModelCatalog` and a few classes but not this + * function, its tarball ships `lib/` and no `src/` (so the `./src/*` export + * resolves nothing), and the package is not a dependency here in any case. + * `initializeDefaultModel` (`index.ts:320-331`) calls it after a Sign-in and, + * when it answers false, replaces the user's default model with the account's + * first one. It answers the FIRST configurable provider whose profile, read + * from `settings.describe`'s VALUE at the provider's `settingsPath`, names an + * `apiKeyEnv` the credentials service reports as configured — so it is the + * reader F20 changes. + * + * It keeps upstream's `credentialRef(ref)` brand as a grammar check that + * throws the same `TypeError`, so a profile whose `apiKeyEnv` is outside the + * grammar fails here as it fails upstream rather than reading "not + * configured". The deliberate differences: it answers the provider id instead + * of `true`, and it omits upstream's `RemoteError` for a missing `settings` or + * `credentials` service, which tier B always provides. * - * @param tier - the booted host. + * @param tier - the booted host (only `ctx.settings`, `llm` and + * `credentials.service` are read). * @returns that provider's id (`true` upstream), or `undefined` (`false`). + * @throws {TypeError} for an `apiKeyEnv` outside `credentialRef`'s grammar. */ export const providerWithApiKey = async ( - tier: TierB + tier: Pick ): Promise => { const describeRef = tier.credentials.service.describe as unknown as ( ref: string @@ -1406,6 +1432,11 @@ export const providerWithApiKey = async ( } const ref: unknown = Reflect.get(profile, "apiKeyEnv"); if (typeof ref === "string" && ref.length > 0) { + if (!CREDENTIAL_REF_PATTERN.test(ref)) { + throw new TypeError( + `credential ref "${ref}" must match ${String(CREDENTIAL_REF_PATTERN)}` + ); + } const described = await describeRef(ref); if (described.configured) { return provider.provider; From 1f48272bdc4630c624f1a95460618e3d4fbaee43 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 14:37:27 +0000 Subject: [PATCH 03/11] test: follow the vendor catalog refresh The vendor renamed the Go plane's Gemini model from gemini-4-argon-contributor to gemini-4-argon. The tier-B fixtures (S0_PROFILE, S0_TODAY, CHARACTERIZED_PROFILE through S0, and todayRowsFor's pi-ai-first override) declared the old id as the Go gap id: one the plugin's catalog describes and pi 1.1.0 cannot place. After the refresh the catalog no longer described it. They now name the renamed id, which still plays that role because pi 1.1.0's opencode-go table has no gemini-4-argon either. The host-shapes and editor-contract cases follow the same fixture. The pi 1.3.0 note in mount-protocols names that pi's own row, so it stays as it is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- test/host-characterization.test.ts | 30 +++++++++++++----------------- test/host-editor-contracts.test.ts | 2 +- test/host-shapes.test.ts | 4 ++-- test/support/host.ts | 14 +++++++------- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/test/host-characterization.test.ts b/test/host-characterization.test.ts index 1125f822..b878df58 100644 --- a/test/host-characterization.test.ts +++ b/test/host-characterization.test.ts @@ -291,9 +291,7 @@ describe( it("fixture: the gap ids are gaps on pi 1.1.0, the unknown id is unknown everywhere, and the known ids are known", () => { expect(piKnows("opencode", "gpt-5.3-codex-spark")).toBeUndefined(); - expect( - piKnows("opencode-go", "gemini-4-argon-contributor") - ).toBeUndefined(); + expect(piKnows("opencode-go", "gemini-4-argon")).toBeUndefined(); expect(piKnows("opencode-go", "minimax-m3.1-flash")).toBeUndefined(); expect(piKnows("opencode", "deepseek-v4-pro")).toBe("openai-completions"); expect(piKnows("opencode", "gemini-3-flash")).toBe( @@ -487,7 +485,7 @@ describe( ).toMatch(/model "gpt-5\.3-codex-spark" needs an api/u); expect( directory.find((entry) => entry.provider === "opencode-go")?.error - ).toMatch(/model "gemini-4-argon-contributor" needs an api/u); + ).toMatch(/model "gemini-4-argon" needs an api/u); }); it("[INTENDED_CHANGE] keyless listing: before a key is captured, the listing patch hides every paid model on both planes (D21)", async () => { @@ -510,7 +508,7 @@ describe( } expect(listed).toEqual(S0_TODAY.keyedListing); expect(listed.opencode).not.toContain("gpt-5.3-codex-spark"); - expect(listed["opencode-go"]).not.toContain("gemini-4-argon-contributor"); + expect(listed["opencode-go"]).not.toContain("gemini-4-argon"); expect(listed["opencode-go"]).not.toContain("minimax-m3.1-flash"); for (const plane of PLANES) { expect(listed[plane]).not.toContain(UNKNOWN_ID); @@ -607,16 +605,16 @@ describe( expect(drained.finish).toEqual({ code: "INVALID_CONFIG", kind: "error" }); }); - it("[KNOWN_DEFECT] Go gemini-4-argon-contributor: resolveCallConfig refuses it (INVALID_CONFIG) while the turn is served on the Go Google base through the bridge", async () => { - expect( - await callConfigOf("opencode-go", "gemini-4-argon-contributor") - ).toBe("INVALID_CONFIG"); + it("[KNOWN_DEFECT] Go gemini-4-argon: resolveCallConfig refuses it (INVALID_CONFIG) while the turn is served on the Go Google base through the bridge", async () => { + expect(await callConfigOf("opencode-go", "gemini-4-argon")).toBe( + "INVALID_CONFIG" + ); const { drained, requests } = await sentBy( "opencode-go", - "gemini-4-argon-contributor" + "gemini-4-argon" ); expect(requests.map((request) => request.url)).toEqual( - S0_TODAY.rows["opencode-go/gemini-4-argon-contributor"].sent + S0_TODAY.rows["opencode-go/gemini-4-argon"].sent ); expect(requests[0]?.headers["x-goog-api-key"]).toBe("go-test-key".length); // pi maps a Google 401 to its own code, not AUTH (as with pi alone). @@ -654,7 +652,7 @@ describe( ); const cleared = readFileSync(profile.patchPath, "utf8"); expect(cleared).not.toContain("gpt-5.3-codex-spark"); - expect(cleared).toContain("gemini-4-argon-contributor"); + expect(cleared).toContain("gemini-4-argon"); await expect( ctx.configEditor.edit(piEntry, appendTo("opencode", SAVE_CANDIDATE)) ).rejects.toThrow(S0_TODAY.saveRejectedUnknown); @@ -727,7 +725,7 @@ describe("characterization · the apply order (no host)", () => { // when P-R1 flips this row it drops todayRowsFor's pi-ai-first override in // the same diff, and this case becomes "[BASELINE] no facet depends on the // apply order" (CONTRIBUTING.md, "Tags"). - it("[KNOWN_DEFECT] the apply order decides exactly one facet today: Go gemini-4-argon-contributor's callConfig is INVALID_CONFIG in the published order and \"ok\" with pi-ai first (P-R1 drops todayRowsFor's pi-ai-first override with this row's flip)", () => { + it("[KNOWN_DEFECT] the apply order decides exactly one facet today: Go gemini-4-argon's callConfig is INVALID_CONFIG in the published order and \"ok\" with pi-ai first (P-R1 drops todayRowsFor's pi-ai-first override with this row's flip)", () => { const piAiFirst = todayRowsFor("pi-ai-first"); const moved = ROW_KEYS.flatMap((key) => FACETS.filter( @@ -736,10 +734,8 @@ describe("characterization · the apply order (no host)", () => { JSON.stringify(expectedFacet(facet, key)) ).map((facet) => `${key} · ${facet}`) ); - expect(moved).toEqual([ - "opencode-go/gemini-4-argon-contributor · callConfig", - ]); - const key = "opencode-go/gemini-4-argon-contributor"; + expect(moved).toEqual(["opencode-go/gemini-4-argon · callConfig"]); + const key = "opencode-go/gemini-4-argon"; expect(expectedFacet("callConfig", key)).toBe("INVALID_CONFIG"); expect(expectedFacet("callConfig", key, piAiFirst)).toBe("ok"); }); diff --git a/test/host-editor-contracts.test.ts b/test/host-editor-contracts.test.ts index 3c042fd1..2f2698f5 100644 --- a/test/host-editor-contracts.test.ts +++ b/test/host-editor-contracts.test.ts @@ -57,7 +57,7 @@ const knownProviders = (): Record> => { const providers = s0Providers(); const gaps = new Set([ "gpt-5.3-codex-spark", - "gemini-4-argon-contributor", + "gemini-4-argon", "minimax-m3.1-flash", ]); for (const plane of ["opencode", "opencode-go"]) { diff --git a/test/host-shapes.test.ts b/test/host-shapes.test.ts index 9939b43f..2f2dc1f1 100644 --- a/test/host-shapes.test.ts +++ b/test/host-shapes.test.ts @@ -301,7 +301,7 @@ describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,4 ...( profileOf(tier, "opencode-go").modelErrors as Map ).keys(), - ]).toEqual(["gemini-4-argon-contributor", "minimax-m3.1-flash"]); + ]).toEqual(["gemini-4-argon", "minimax-m3.1-flash"]); }); it("F4 — a stored id pi cannot place is not listed, and a turn on it is INVALID_CONFIG with no request (index.ts:171; adapter.ts:271-311)", async () => { @@ -334,7 +334,7 @@ describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,4 ]); for (const [route, model] of [ ["opencode", "gpt-5.3-codex-spark"], - ["opencode-go", "gemini-4-argon-contributor"], + ["opencode-go", "gemini-4-argon"], ["opencode-go", "minimax-m3.1-flash"], ] as const) { const drained = await drainStream(mounted.llm, route, model); diff --git a/test/support/host.ts b/test/support/host.ts index ca125a74..77222897 100644 --- a/test/support/host.ts +++ b/test/support/host.ts @@ -428,7 +428,7 @@ export const S0_PROFILE = { { id: "deepseek-v4-pro" }, { id: "muse-spark-1.2-contributor" }, { id: "minimax-m3" }, - { id: "gemini-4-argon-contributor" }, + { id: "gemini-4-argon" }, { id: "minimax-m3.1-flash" }, ], }, @@ -654,12 +654,12 @@ export const S0_TODAY = { sent: [`${ZEN}/inference/go/anthropic/v1/messages?beta=true`], tags: tags(D, B, B), }, - "opencode-go/gemini-4-argon-contributor": { + "opencode-go/gemini-4-argon": { callConfig: "INVALID_CONFIG", info: { context: 1_048_576, efforts: ["low", "medium", "high"] }, finish: "PI_AI_ERROR", sent: [ - `${ZEN}/inference/go/google/v1beta/models/gemini-4-argon-contributor:streamGenerateContent?alt=sse`, + `${ZEN}/inference/go/google/v1beta/models/gemini-4-argon:streamGenerateContent?alt=sse`, ], tags: tags(B, D, B), }, @@ -705,7 +705,7 @@ export const ZEN_UNDESCRIBED = ["gpt-5.3-codex-spark", UNKNOWN_ID] as const; /** * TODAY's rows for an apply order. `S0_TODAY.rows` is the `plugin-first` * table; with `pi-ai-first` the adapter decoration wraps pi-ai's plane adapter - * and exactly one facet moves — Go gemini-4-argon-contributor validates — + * and exactly one facet moves — Go gemini-4-argon validates — * measured in `test/host-characterization.test.ts`, which generates one test * per row and facet for each order, titled with that order's tags. The moved * facet is `BASELINE` in this table: its `"ok"` is already what P-R1 keeps, so @@ -730,11 +730,11 @@ export const todayRowsFor = ( } return { ...S0_TODAY.rows, - "opencode-go/gemini-4-argon-contributor": { - ...S0_TODAY.rows["opencode-go/gemini-4-argon-contributor"], + "opencode-go/gemini-4-argon": { + ...S0_TODAY.rows["opencode-go/gemini-4-argon"], callConfig: "ok", tags: { - ...S0_TODAY.rows["opencode-go/gemini-4-argon-contributor"].tags, + ...S0_TODAY.rows["opencode-go/gemini-4-argon"].tags, callConfig: B, }, }, From 01012390a5445648adb893505563130c56fa5853 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 16:01:07 +0000 Subject: [PATCH 04/11] test: characterize the real host on both harness channels, newest and floor The rows were measured on pi 1.1.0 with the harness patch, a host no npm user runs. Each tier file now runs on whichever host cell is installed (either channel's newest pi, or a channel's floor), and a row carries one expectation per cell where the hosts differ: pi's catalog bases, the ids pi can place, F10 and pi-ai's own defaults. Facets pi 1.5.0 already serves the way P-R1 keeps are BASELINE on that cell. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- test/host-characterization.test.ts | 283 ++++++++++++++++--------- test/host-credentials.test.ts | 30 ++- test/host-editor-contracts.test.ts | 5 +- test/host-shapes.test.ts | 319 +++++++++++++++++++++-------- test/host-waterfall.test.ts | 8 +- test/keyless-matrix.test.ts | 63 +++++- test/support/host.ts | 267 +++++++++++++++++++++--- test/support/keyless-matrix.ts | 16 +- 8 files changed, 746 insertions(+), 245 deletions(-) diff --git a/test/host-characterization.test.ts b/test/host-characterization.test.ts index b878df58..cf4b46e4 100644 --- a/test/host-characterization.test.ts +++ b/test/host-characterization.test.ts @@ -1,5 +1,7 @@ /** - * TODAY's plugin on the real host, characterized (tier B, pi 1.1.0). + * TODAY's plugin on the real host, characterized (tier B), on the host cell + * the install under test is (`HOST_CELL`: either channel's newest pi, or a + * channel's floor after `scripts/use-channel.ts --floor`). * * The plugin is mounted as it ships — its row carries no config, so the * shipped defaults apply: the hidden ids in `providers`, `enrichModels` on, @@ -12,6 +14,11 @@ * - `[INTENDED_CHANGE]` is right today, and a phase changes it on purpose; * - `[BASELINE]` must survive every phase. * + * Where the hosts differ, a row carries one expectation per cell + * (`S0_TODAY_BY_CELL`, `forHost`), and a title names the cell's tag: a facet + * pi 1.5.0 already serves the way P-R1 keeps is `[BASELINE]` on `newest` and + * still `[INTENDED_CHANGE]` or `[KNOWN_DEFECT]` on a floor. + * * The per-model expectations live in `S0_TODAY` (`support/host.ts`), which * the binary e2e reads too, so the in-process host and the published binary * cannot drift apart unnoticed. Each model's three facets (model info, the @@ -34,6 +41,7 @@ import { readFileSync } from "node:fs"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CHANNELS } from "./support/host-versions.ts"; import { adaptersOf, type ApplyOrder, @@ -43,13 +51,21 @@ import { characterizedProviders, drainStream, fakeCredentials, + forHost, + HOST_CELL, + HOST_CELLS, + PI_GAPS, + PI_OF_CELL, PLANES, piKnows, providerWithApiKey, recordingFetch, type RecordingFetch, type RowFacet, + type RowTag, S0_TODAY, + S0_TODAY_BY_CELL, + type S0RowKey, s0Providers, SAVE_BLOCKED_BY_ANY_UNDESCRIBED, SAVE_CANDIDATE, @@ -68,7 +84,10 @@ import { thinkingLevelMapFor, } from "./support/src.ts"; -type RowKey = keyof typeof S0_TODAY.rows; +type RowKey = S0RowKey; + +/** The pi the install under test runs, for the titles. */ +const PI = PI_OF_CELL[HOST_CELL]; let tier: TierB | undefined; let wire: RecordingFetch; @@ -233,6 +252,34 @@ const dropFrom = const FACETS = ["info", "callConfig", "wire"] as const; +/** The facets whose value differs between a cell's two apply orders. */ +const movedByOrder = (cell: (typeof HOST_CELLS)[number]): string[] => { + const published = todayRowsFor("plugin-first", cell); + const piAiFirst = todayRowsFor("pi-ai-first", cell); + return ROW_KEYS.flatMap((key) => + FACETS.filter( + (facet) => + JSON.stringify(expectedFacet(facet, key, piAiFirst)) !== + JSON.stringify(expectedFacet(facet, key, published)) + ).map((facet) => `${key} · ${facet}`) + ); +}; + +/** + * Where completions go, per cell: pi's catalog base, which moved under the + * plugin twice (the legacy family on 1.1.0, the root on 1.3.0, the + * per-protocol family P-R1 targets on 1.5.0). + */ +const COMPLETIONS_FAMILY = { + "alpha-floor": "the root family, /inference(/go)/v1", + newest: "the per-protocol family, /inference(/go)/openai/v1", + "rc-floor": "the legacy family, /zen(/go)/v1", +} as const; + +/** The completions wire facet's tag on this cell (both planes agree). */ +const COMPLETIONS_TAG: RowTag = + S0_TODAY.rows["opencode/deepseek-v4-pro"].tags.wire; + /** * One generated case per row and facet, titled with the facet's own tag in * that order's table, so a phase's diff names every facet it moves in both @@ -284,15 +331,22 @@ const expectSignInReadsTheZenPlane = async (): Promise => { }; describe( - "characterization · today's plugin on pi 1.1.0, the published host's order (tier B)", + `characterization · today's plugin on ${HOST_CELL} (pi ${PI}), the published host's order (tier B)`, { timeout: 30_000 }, () => { beforeEach(() => bootS0("plugin-first"), 30_000); - it("fixture: the gap ids are gaps on pi 1.1.0, the unknown id is unknown everywhere, and the known ids are known", () => { - expect(piKnows("opencode", "gpt-5.3-codex-spark")).toBeUndefined(); - expect(piKnows("opencode-go", "gemini-4-argon")).toBeUndefined(); - expect(piKnows("opencode-go", "minimax-m3.1-flash")).toBeUndefined(); + it(`fixture: this cell's gap ids are gaps on pi ${PI} and every other declared id is placed, the unknown id is unknown everywhere, and the cells' tables agree on their keys`, () => { + for (const plane of PLANES) { + for (const id of CHARACTERIZED_MODELS[plane]) { + if (id === UNKNOWN_ID) { + continue; + } + expect(piKnows(plane, id), `${plane}/${id}`).toEqual( + PI_GAPS[plane].includes(id) ? undefined : expect.any(String) + ); + } + } expect(piKnows("opencode", "deepseek-v4-pro")).toBe("openai-completions"); expect(piKnows("opencode", "gemini-3-flash")).toBe( "google-generative-ai" @@ -319,6 +373,14 @@ describe( ) ); expect(INTERNAL_ROUTES).toHaveLength(6); + for (const cell of HOST_CELLS) { + expect(Object.keys(S0_TODAY_BY_CELL[cell].rows), cell).toEqual( + ROW_KEYS + ); + } + // `newest` is one column for both channels only while they resolve one + // pi; the day they do not, split it. + expect(CHANNELS.rc.pi).toBe(CHANNELS.alpha.pi); }); it("fixture: each row's context and efforts are the bundled catalog's, so a catalog refresh that moves a fixture model fails here by name", () => { @@ -340,10 +402,12 @@ describe( const INFO_NOT_FROM_CATALOG = new Set(); for (const key of INFO_NOT_FROM_CATALOG) { expect(ROW_KEYS, `${key} is not a characterized row`).toContain(key); - expect( - S0_TODAY.rows[key].tags.info, - `${key}: a BASELINE info facet cannot be exempted from the catalog` - ).not.toBe("BASELINE"); + for (const cell of HOST_CELLS) { + expect( + S0_TODAY_BY_CELL[cell].rows[key].tags.info, + `${key}: a BASELINE info facet cannot be exempted from the catalog` + ).not.toBe("BASELINE"); + } } let compared = 0; for (const key of ROW_KEYS) { @@ -361,6 +425,14 @@ describe( "either a catalog refresh moved it (update the row in " + "test/support/host.ts), or a phase flipped this facet off the " + "catalog (add the key to INFO_NOT_FROM_CATALOG here)"; + // No cell's pi moves an info facet: each comes from the plugin's + // catalog, so every cell must agree with it. + for (const cell of HOST_CELLS) { + expect( + S0_TODAY_BY_CELL[cell].rows[key].info, + `${cell} ${key}` + ).toEqual(S0_TODAY.rows[key].info); + } const info = S0_TODAY.rows[key].info as { context?: number; efforts?: readonly string[]; @@ -383,14 +455,18 @@ describe( expect(compared).toBeGreaterThan(0); }); - it("fixture: the binary e2e's cell-aware helpers — the floating save-block pattern names any undescribed id and only those, todayRowsFor refuses an order it does not know, and a facet its pi-ai-first table moves is retagged with it, from KNOWN_DEFECT to BASELINE", () => { - // The pinned message is one the floating pattern accepts, so the two - // cells cannot disagree on a pinned run. - const pinned = + it("fixture: the binary e2e's cell-aware helpers — the newer-pi save-block pattern names any undescribed id and only those, todayRowsFor refuses an order it does not know, and a facet its pi-ai-first table moves is retagged with it, from KNOWN_DEFECT to BASELINE", () => { + // Every recorded cell's message is one the newer-pi pattern accepts, so + // the two cannot disagree on a run whose pi is the record's. + const recorded = 'provider "opencode" model "gpt-5.3-codex-spark" needs an api'; - expect(pinned).toMatch(S0_TODAY.saveBlockedByStoredGap); - expect(pinned).toMatch(SAVE_BLOCKED_BY_ANY_UNDESCRIBED); - // A floating pi that learned gpt-5.3-codex-spark names the next one. + for (const cell of HOST_CELLS) { + expect(recorded, cell).toMatch( + S0_TODAY_BY_CELL[cell].saveBlockedByStoredGap + ); + } + expect(recorded).toMatch(SAVE_BLOCKED_BY_ANY_UNDESCRIBED); + // A newer pi that learned gpt-5.3-codex-spark names the next one. expect(`provider "opencode" model "${UNKNOWN_ID}" needs an api`).toMatch( SAVE_BLOCKED_BY_ANY_UNDESCRIBED ); @@ -401,7 +477,9 @@ describe( ); } for (const order of APPLY_ORDERS) { - expect(Object.keys(todayRowsFor(order))).toEqual(ROW_KEYS); + for (const cell of HOST_CELLS) { + expect(Object.keys(todayRowsFor(order, cell))).toEqual(ROW_KEYS); + } } // Structural, so no phase has to edit it: a facet whose VALUE differs // between the orders is exactly a facet whose TAG differs, and each one @@ -409,30 +487,26 @@ describe( // what P-R1 keeps. Which facets those are today is pinned by the // [KNOWN_DEFECT] apply-order case below; once P-R1 collapses the // override, both sets are empty and this still holds. - const piAiFirst = todayRowsFor("pi-ai-first"); - const moved = ROW_KEYS.flatMap((key) => - FACETS.filter( - (facet) => - JSON.stringify(expectedFacet(facet, key, piAiFirst)) !== - JSON.stringify(expectedFacet(facet, key)) - ).map((facet) => `${key} · ${facet}`) - ); - const retagged = ROW_KEYS.flatMap((key) => - FACETS.filter( - (facet) => - piAiFirst[key].tags[facet] !== S0_TODAY.rows[key].tags[facet] - ).map((facet) => `${key} · ${facet}`) - ); - expect(retagged).toEqual(moved); - for (const key of ROW_KEYS) { - for (const facet of FACETS) { - if (moved.includes(`${key} · ${facet}`)) { - expect(S0_TODAY.rows[key].tags[facet], `${key} · ${facet}`).toBe( - "KNOWN_DEFECT" - ); - expect(piAiFirst[key].tags[facet], `${key} · ${facet}`).toBe( - "BASELINE" - ); + for (const cell of HOST_CELLS) { + const published = todayRowsFor("plugin-first", cell); + const piAiFirst = todayRowsFor("pi-ai-first", cell); + const moved = movedByOrder(cell); + const retagged = ROW_KEYS.flatMap((key) => + FACETS.filter( + (facet) => piAiFirst[key].tags[facet] !== published[key].tags[facet] + ).map((facet) => `${key} · ${facet}`) + ); + expect(retagged, cell).toEqual(moved); + for (const key of ROW_KEYS) { + for (const facet of FACETS) { + if (moved.includes(`${key} · ${facet}`)) { + expect(published[key].tags[facet], `${cell} ${key}`).toBe( + "KNOWN_DEFECT" + ); + expect(piAiFirst[key].tags[facet], `${cell} ${key}`).toBe( + "BASELINE" + ); + } } } } @@ -479,13 +553,15 @@ describe( settingsPath: ["providers", plane], }); } - // The directory reports each plane's first stored gap id. - expect( - directory.find((entry) => entry.provider === "opencode")?.error - ).toMatch(/model "gpt-5\.3-codex-spark" needs an api/u); - expect( - directory.find((entry) => entry.provider === "opencode-go")?.error - ).toMatch(/model "gemini-4-argon" needs an api/u); + // The directory reports each plane's first id the installed pi cannot + // place: its first gap id, else the declared unknown id. + for (const plane of PLANES) { + const first = PI_GAPS[plane][0] ?? UNKNOWN_ID; + expect( + directory.find((entry) => entry.provider === plane)?.error, + plane + ).toContain(`model "${first}" needs an api`); + } }); it("[INTENDED_CHANGE] keyless listing: before a key is captured, the listing patch hides every paid model on both planes (D21)", async () => { @@ -507,10 +583,10 @@ describe( listed[plane] = models.map((model) => model.id); } expect(listed).toEqual(S0_TODAY.keyedListing); - expect(listed.opencode).not.toContain("gpt-5.3-codex-spark"); - expect(listed["opencode-go"]).not.toContain("gemini-4-argon"); - expect(listed["opencode-go"]).not.toContain("minimax-m3.1-flash"); for (const plane of PLANES) { + for (const gap of PI_GAPS[plane]) { + expect(listed[plane]).not.toContain(gap); + } expect(listed[plane]).not.toContain(UNKNOWN_ID); } }); @@ -535,8 +611,8 @@ describe( "minimax-m3" ); const efforts = info.reasoning?.efforts?.map((effort) => effort.id) ?? []; - expect(efforts).toEqual( - S0_TODAY.rows["opencode-go/minimax-m3"].info.efforts + expect({ context: info.context?.contextWindow, efforts }).toEqual( + S0_TODAY.rows["opencode-go/minimax-m3"].info ); expect(efforts).toEqual( expect.arrayContaining(["minimal", "xhigh", "max"]) @@ -544,13 +620,12 @@ describe( expect(efforts).not.toContain("off"); }); - it("[INTENDED_CHANGE] completions models dispatch to pi 1.1.0's legacy bases", async () => { - for (const [route, wireUrl] of [ - ["opencode", "https://opencode.ai/zen/v1/chat/completions"], - ["opencode-go", "https://opencode.ai/zen/go/v1/chat/completions"], - ] as const) { + it(`[${COMPLETIONS_TAG}] completions models dispatch to pi ${PI}'s own catalog base (${forHost(COMPLETIONS_FAMILY)})`, async () => { + for (const route of PLANES) { const { drained, requests } = await sentBy(route, "deepseek-v4-pro"); - expect(requests.map((request) => request.url)).toEqual([wireUrl]); + expect(requests.map((request) => request.url)).toEqual( + S0_TODAY.rows[`${route}/deepseek-v4-pro`].sent + ); expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); } }); @@ -586,28 +661,34 @@ describe( ]); }); - it("[KNOWN_DEFECT] Go minimax-m3.1-flash answers INVALID_CONFIG 'needs an api' and sends nothing (F4)", async () => { - const { llm } = booted(); - await expect( - llm.resolveCallConfig({ - model: "minimax-m3.1-flash", - provider: "opencode-go", - } as never) - ).rejects.toMatchObject({ - code: "INVALID_CONFIG", - message: expect.stringMatching(/"minimax-m3\.1-flash" needs an api/u), - }); - const { drained, requests } = await sentBy( - "opencode-go", - "minimax-m3.1-flash" - ); - expect(requests).toEqual([]); - expect(drained.finish).toEqual({ code: "INVALID_CONFIG", kind: "error" }); - }); + it.runIf(PI_GAPS["opencode-go"].includes("minimax-m3.1-flash"))( + `[KNOWN_DEFECT] Go minimax-m3.1-flash answers INVALID_CONFIG 'needs an api' and sends nothing (F4; a gap on pi ${PI}, placed on newest, where its facet rows are BASELINE)`, + async () => { + const { llm } = booted(); + await expect( + llm.resolveCallConfig({ + model: "minimax-m3.1-flash", + provider: "opencode-go", + } as never) + ).rejects.toMatchObject({ + code: "INVALID_CONFIG", + message: expect.stringMatching(/"minimax-m3\.1-flash" needs an api/u), + }); + const { drained, requests } = await sentBy( + "opencode-go", + "minimax-m3.1-flash" + ); + expect(requests).toEqual([]); + expect(drained.finish).toEqual({ + code: "INVALID_CONFIG", + kind: "error", + }); + } + ); - it("[KNOWN_DEFECT] Go gemini-4-argon: resolveCallConfig refuses it (INVALID_CONFIG) while the turn is served on the Go Google base through the bridge", async () => { + it(`[${S0_TODAY.rows["opencode-go/gemini-4-argon"].tags.callConfig}] Go gemini-4-argon: resolveCallConfig answers ${S0_TODAY.rows["opencode-go/gemini-4-argon"].callConfig} on pi ${PI} (INVALID_CONFIG where pi lacks the id) while the turn is served on the Go Google base`, async () => { expect(await callConfigOf("opencode-go", "gemini-4-argon")).toBe( - "INVALID_CONFIG" + S0_TODAY.rows["opencode-go/gemini-4-argon"].callConfig ); const { drained, requests } = await sentBy( "opencode-go", @@ -671,13 +752,11 @@ describe( await expectSignInReadsTheZenPlane(); }); - it("[INTENDED_CHANGE] the adapter decoration runs, and the base it repairs never reaches the wire", async () => { + it("[INTENDED_CHANGE] the adapter decoration runs, and the base it repairs is not what dispatch uses: the wire carries pi's own catalog base", async () => { const adapter = planeAdapter(); expect(Object.hasOwn(adapter, "prepareCall")).toBe(true); - for (const [route, wireUrl] of [ - ["opencode", "https://opencode.ai/zen/v1/chat/completions"], - ["opencode-go", "https://opencode.ai/zen/go/v1/chat/completions"], - ] as const) { + for (const route of PLANES) { + const [wireUrl] = S0_TODAY.rows[`${route}/deepseek-v4-pro`].sent; // Measured: the Go plane's repaired base is ZEN's. pi-ai serves both // planes from ONE adapter instance, which decoratePlaneAdapters wraps // once per route, so the Go wrapper runs over the Zen one and keeps the @@ -690,10 +769,11 @@ describe( }); const { drained, requests } = await sentBy(route, "deepseek-v4-pro"); expect(requests.map((request) => request.url)).toEqual([wireUrl]); - // Measured, not compared as literals: the URL the turn sent does not - // start with the base the decorated adapter prepared. + // Measured, not compared as literals: the URL the turn sent starts + // with the base the decorated adapter prepared only where pi's own + // catalog base happens to be the same one (Zen on newest). expect(requests[0]?.url.startsWith(String(prepared.baseUrl))).toBe( - false + String(wireUrl).startsWith(decorated) ); expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); } @@ -725,19 +805,18 @@ describe("characterization · the apply order (no host)", () => { // when P-R1 flips this row it drops todayRowsFor's pi-ai-first override in // the same diff, and this case becomes "[BASELINE] no facet depends on the // apply order" (CONTRIBUTING.md, "Tags"). - it("[KNOWN_DEFECT] the apply order decides exactly one facet today: Go gemini-4-argon's callConfig is INVALID_CONFIG in the published order and \"ok\" with pi-ai first (P-R1 drops todayRowsFor's pi-ai-first override with this row's flip)", () => { - const piAiFirst = todayRowsFor("pi-ai-first"); - const moved = ROW_KEYS.flatMap((key) => - FACETS.filter( - (facet) => - JSON.stringify(expectedFacet(facet, key, piAiFirst)) !== - JSON.stringify(expectedFacet(facet, key)) - ).map((facet) => `${key} · ${facet}`) - ); - expect(moved).toEqual(["opencode-go/gemini-4-argon · callConfig"]); - const key = "opencode-go/gemini-4-argon"; - expect(expectedFacet("callConfig", key)).toBe("INVALID_CONFIG"); - expect(expectedFacet("callConfig", key, piAiFirst)).toBe("ok"); + it("[KNOWN_DEFECT] the apply order decides exactly one facet on the floor cells: Go gemini-4-argon's callConfig is INVALID_CONFIG in the published order and \"ok\" with pi-ai first; on newest pi places the id and nothing depends on the order (P-R1 drops todayRowsFor's pi-ai-first override with this row's flip)", () => { + for (const cell of ["alpha-floor", "rc-floor"] as const) { + expect(movedByOrder(cell), cell).toEqual([ + "opencode-go/gemini-4-argon · callConfig", + ]); + const key = "opencode-go/gemini-4-argon"; + expect(todayRowsFor("plugin-first", cell)[key].callConfig).toBe( + "INVALID_CONFIG" + ); + expect(todayRowsFor("pi-ai-first", cell)[key].callConfig).toBe("ok"); + } + expect(movedByOrder("newest")).toEqual([]); }); }); diff --git a/test/host-credentials.test.ts b/test/host-credentials.test.ts index 504b8ab4..6c0e1025 100644 --- a/test/host-credentials.test.ts +++ b/test/host-credentials.test.ts @@ -20,6 +20,7 @@ import { drainStream, fakeCredentials, fakeLaunchEnvironment, + forHost, mountTierA, recordingFetch, type TierA, @@ -42,7 +43,7 @@ interface PiAuthResult { /** * pi's own auth answer for a route, through the collection pi-ai's adapter * built for its current snapshot (adapter.ts:251-259, private) and pi's public - * `Models.getAuth` (pi 1.1.0 `dist/models.d.ts`). It is the one place a stored + * `Models.getAuth` (pi's `dist/models.d.ts`). It is the one place a stored * record's `env` is observable on a plane: pi forwards it as the result's * `env` (`dist/auth/helpers.js`), and no plane request reads it back. * @@ -65,9 +66,20 @@ const piAuthOf = async ( : { env: result.env, source: result.source }; }; -/** One completions model on Zen, which pi 1.1.0 serves at its legacy base. */ +/** One completions model on Zen. */ const ZEN_COMPLETIONS = { models: [{ id: "deepseek-v4-pro" }] }; -const LEGACY_COMPLETIONS = "https://opencode.ai/zen/v1/chat/completions"; + +/** + * Where pi sends it, on each host cell: pi's own catalog base, which moved + * from the legacy family (1.1.0) to the root one (1.3.0) and to the + * per-protocol one (1.5.0). The credential paths these cases pin do not + * depend on the base; the URL only shows the turn reached the wire. + */ +const ZEN_COMPLETIONS_URL = forHost({ + "alpha-floor": "https://opencode.ai/inference/v1/chat/completions", + newest: "https://opencode.ai/inference/openai/v1/chat/completions", + "rc-floor": "https://opencode.ai/zen/v1/chat/completions", +}); describe("host · pi-ai's credential paths (tier A)", () => { it("maps an api-key record under llm-pi-ai/opencode onto the wire with no apiKeyEnv (auth.ts:74-84,141-148)", async () => { @@ -82,7 +94,7 @@ describe("host · pi-ai's credential paths (tier A)", () => { }); const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); expect(wire.requests.map((request) => request.url)).toEqual([ - LEGACY_COMPLETIONS, + ZEN_COMPLETIONS_URL, ]); expect(wire.requests[0]?.headers.authorization).toBe( "Bearer stored-k".length @@ -121,7 +133,7 @@ describe("host · pi-ai's credential paths (tier A)", () => { }); const drained = await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); expect(wire.requests.map((request) => request.url)).toEqual([ - LEGACY_COMPLETIONS, + ZEN_COMPLETIONS_URL, ]); // The stored key's length, not the environment's (a different length). expect(wire.requests[0]?.headers.authorization).toBe( @@ -138,8 +150,8 @@ describe("host · pi-ai's credential paths (tier A)", () => { }); it("hands a non-api-key record's payload to pi verbatim: on a plane, whose pi auth has no OAuth, the turn fails PI_AI_ERROR with no request and no fallback to the launch environment (auth.ts:74-84)", async () => { - // Not a record a plane's Sign-in writes (pi 1.1.0's planes offer an API - // key only); pinned so P-R1's store, which copies this mapping, answers + // Not a record a plane's Sign-in writes (pi's planes offer an API key + // only); pinned so P-R1's store, which copies this mapping, answers // the same for one a user or another plane left behind. const wire = recordingFetch(); vi.stubGlobal("fetch", wire.fetch); @@ -214,7 +226,7 @@ describe("host · pi-ai's credential paths (tier A)", () => { expect(listed.map((model) => model.id)).toEqual(["deepseek-v4-pro"]); await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); expect(wire.requests.map((request) => request.url)).toEqual([ - LEGACY_COMPLETIONS, + ZEN_COMPLETIONS_URL, ]); expect(wire.requests[0]?.headers.authorization).toBe("Bearer ref-k".length); }); @@ -237,7 +249,7 @@ describe("host · pi-ai's credential paths (tier A)", () => { }); await drainStream(tier.llm, "opencode", "deepseek-v4-pro"); expect(wire.requests.map((request) => request.url)).toEqual([ - LEGACY_COMPLETIONS, + ZEN_COMPLETIONS_URL, ]); expect(wire.requests[0]?.headers.authorization).toBe("Bearer env-k".length); // The stored record is read first, then the reference, then the diff --git a/test/host-editor-contracts.test.ts b/test/host-editor-contracts.test.ts index 2f2698f5..5382cfe6 100644 --- a/test/host-editor-contracts.test.ts +++ b/test/host-editor-contracts.test.ts @@ -52,7 +52,10 @@ const prependStrip = (root: Context): void => { root.on("internal/config", stripPlanesForPiAi, { prepend: true }); }; -/** S0 without the ids pi 1.1.0 cannot place, so pi-ai's save check passes. */ +/** + * S0 without the ids the floor cells' pi cannot place (`PI_GAPS_BY_CELL`), so + * pi-ai's save check passes on every cell with the same profile. + */ const knownProviders = (): Record> => { const providers = s0Providers(); const gaps = new Set([ diff --git a/test/host-shapes.test.ts b/test/host-shapes.test.ts index 2f2dc1f1..a404d0ff 100644 --- a/test/host-shapes.test.ts +++ b/test/host-shapes.test.ts @@ -9,6 +9,12 @@ * real package, with the host file and line it reads. A host bump that moves * one turns this file red before any routing code depends on it. * + * It runs on whichever host cell is installed (`HOST_CELL`: either channel's + * newest pi, or a channel's floor). What differs between the cells (pi's + * catalog bases and counts, which ids it can place, whether its Go provider + * carries Google, pi-ai's own defaults per cluster) carries one expectation + * per cell (`forHost`) or per channel (`forChannel`), measured on each. + * * `test/mount-protocols.test.ts` already pins the four `api/*.lazy` factories * against the catalog's SDKs; this file does not repeat it. * @@ -27,12 +33,134 @@ import { adaptersOf, drainStream, fakeCredentials, + forChannel, + forHost, + HOST_CELL, mountTierA, + PI_GAPS, + PI_OF_CELL, + PLANES, recordingFetch, + S0_PROFILE, s0Providers, type TierA, } from "./support/host.ts"; +/** The pi the install under test runs, for the titles. */ +const PI = PI_OF_CELL[HOST_CELL]; + +/** Per protocol: the bases pi's catalog files a plane's models under, and how many. */ +type ByProtocol = Record; + +const ZEN_BASE = "https://opencode.ai"; + +/** + * F1/F2, measured per cell: a model-less plane resolves pi's whole catalog. + * The bases moved twice under the plugin: the legacy family on pi 1.1.0, the + * root `/inference(/go)/v1` on 1.3.0, and per-protocol roots on 1.5.0, which + * also dropped four Zen completions models and one Anthropic one. Go gained a + * Google model on 1.3.0 (F10). + */ +const WHOLE_CATALOG: Readonly< + Record< + "alpha-floor" | "newest" | "rc-floor", + { zen: ByProtocol; go: ByProtocol } + > +> = { + "alpha-floor": { + go: { + "anthropic-messages": { bases: [`${ZEN_BASE}/inference/go`], count: 5 }, + "google-generative-ai": { + bases: [`${ZEN_BASE}/inference/go/v1`], + count: 1, + }, + "openai-completions": { + bases: [`${ZEN_BASE}/inference/go/v1`], + count: 18, + }, + "openai-responses": { bases: [`${ZEN_BASE}/inference/go/v1`], count: 5 }, + }, + zen: { + "anthropic-messages": { bases: [`${ZEN_BASE}/inference`], count: 18 }, + "google-generative-ai": { + bases: [`${ZEN_BASE}/inference/v1`], + count: 8, + }, + "openai-completions": { bases: [`${ZEN_BASE}/inference/v1`], count: 28 }, + "openai-responses": { bases: [`${ZEN_BASE}/inference/v1`], count: 30 }, + }, + }, + newest: { + go: { + "anthropic-messages": { + bases: [`${ZEN_BASE}/inference/go/anthropic`], + count: 5, + }, + "google-generative-ai": { + bases: [`${ZEN_BASE}/inference/go/google/v1beta`], + count: 1, + }, + "openai-completions": { + bases: [`${ZEN_BASE}/inference/go/openai/v1`], + count: 19, + }, + "openai-responses": { + bases: [`${ZEN_BASE}/inference/go/openai/v1`], + count: 5, + }, + }, + zen: { + "anthropic-messages": { + bases: [`${ZEN_BASE}/inference/anthropic`], + count: 17, + }, + "google-generative-ai": { + bases: [`${ZEN_BASE}/inference/google/v1beta`], + count: 8, + }, + "openai-completions": { + bases: [`${ZEN_BASE}/inference/openai/v1`], + count: 24, + }, + "openai-responses": { + bases: [`${ZEN_BASE}/inference/openai/v1`], + count: 30, + }, + }, + }, + "rc-floor": { + go: { + "anthropic-messages": { bases: [`${ZEN_BASE}/zen/go`], count: 5 }, + "openai-completions": { bases: [`${ZEN_BASE}/zen/go/v1`], count: 19 }, + "openai-responses": { bases: [`${ZEN_BASE}/zen/go/v1`], count: 6 }, + }, + zen: { + "anthropic-messages": { bases: [`${ZEN_BASE}/zen`], count: 18 }, + "google-generative-ai": { bases: [`${ZEN_BASE}/zen/v1`], count: 7 }, + "openai-completions": { bases: [`${ZEN_BASE}/zen/v1`], count: 28 }, + "openai-responses": { bases: [`${ZEN_BASE}/zen/v1`], count: 30 }, + }, + }, +}; + +/** Collapse one cell's whole catalog onto a single protocol, as F5's route-level `api` does. */ +const collapsed = (catalog: ByProtocol, api: string): ByProtocol => { + const bases = [ + ...new Set(Object.values(catalog).flatMap((row) => row.bases)), + ].toSorted(); + const count = Object.values(catalog).reduce((sum, row) => sum + row.count, 0); + return { [api]: { bases, count } }; +}; + +/** Move every protocol of one cell's catalog onto a single base, as F5's route-level `baseURL` does. */ +const rebased = (catalog: ByProtocol, base: string): ByProtocol => + Object.fromEntries( + Object.entries(catalog).map(([api, row]) => [ + api, + { bases: [base], count: row.count }, + ]) + ); + /** A schemastery node, as far as the role walk reads it. */ interface SchemaNode { readonly type?: string; @@ -156,11 +284,13 @@ describe("host · pi-ai's Config (llm-pi-ai/src/config.ts)", () => { providers: { get: () => Record }; } ).providers.get().x; + // pi-ai's own defaults are the cluster's, not pi's: the alpha cluster + // raised both (llm-pi-ai config.ts:65,68), the rc one keeps the old pair. expect(resolved).toEqual({ compat: { chatTemplateArgs: {}, chatTemplateKwargs: {} }, - defaultContextWindow: 262_144, + defaultContextWindow: forChannel({ alpha: 1_000_000, rc: 262_144 }), defaultInput: ["text"], - defaultMaxTokens: 32_768, + defaultMaxTokens: forChannel({ alpha: 128_000, rc: 32_768 }), headers: {}, maxRequestImageBytes: 20_971_520, modelOverrides: {}, @@ -297,11 +427,12 @@ describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,4 expect(profile.catalogError).toMatch( /model "gpt-5\.3-codex-spark" needs an api/u ); + // The Go gap ids are gaps on the floors only: pi 1.5.0 places both. expect([ ...( profileOf(tier, "opencode-go").modelErrors as Map ).keys(), - ]).toEqual(["gemini-4-argon", "minimax-m3.1-flash"]); + ]).toEqual(PI_GAPS["opencode-go"]); }); it("F4 — a stored id pi cannot place is not listed, and a turn on it is INVALID_CONFIG with no request (index.ts:171; adapter.ts:271-311)", async () => { @@ -321,22 +452,19 @@ describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,4 const models = await mounted.llm.listModels(route); return models.map((model) => model.id); }; - expect(await listed("opencode")).toEqual([ - "deepseek-v4-pro", - "gpt-5.4", - "claude-haiku-5-5", - "gemini-3-flash", - ]); - expect(await listed("opencode-go")).toEqual([ - "deepseek-v4-pro", - "muse-spark-1.2-contributor", - "minimax-m3", - ]); - for (const [route, model] of [ - ["opencode", "gpt-5.3-codex-spark"], - ["opencode-go", "gemini-4-argon"], - ["opencode-go", "minimax-m3.1-flash"], - ] as const) { + // Every declared id the installed pi places is listed, in declaration + // order; none of its gap ids is. + for (const plane of PLANES) { + expect(await listed(plane), plane).toEqual( + S0_PROFILE[plane].models + .map((model) => model.id) + .filter((id) => !PI_GAPS[plane].includes(id)) + ); + } + expect(PI_GAPS.opencode).toEqual(["gpt-5.3-codex-spark"]); + for (const [route, model] of PLANES.flatMap((plane) => + PI_GAPS[plane].map((id) => [plane, id] as const) + )) { const drained = await drainStream(mounted.llm, route, model); expect(drained.finish, `${route}/${model}`).toEqual({ code: "INVALID_CONFIG", @@ -346,36 +474,16 @@ describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,4 expect(wire.requests).toEqual([]); }); - it("F1/F2 — a model-less plane resolves pi 1.1.0's whole catalog, per protocol, on the legacy bases", async () => { + it(`F1/F2 — a model-less plane resolves pi ${PI}'s whole catalog, per protocol, on pi's own bases for this cell`, async () => { tier = await mountTierA({ providers: { opencode: {}, "opencode-go": {} }, }); const zen = profileOf(tier, "opencode"); expect(zen.catalogError).toBeUndefined(); - expect(byProtocol(zen.piProvider)).toEqual({ - "anthropic-messages": { bases: ["https://opencode.ai/zen"], count: 18 }, - "google-generative-ai": { - bases: ["https://opencode.ai/zen/v1"], - count: 7, - }, - "openai-completions": { - bases: ["https://opencode.ai/zen/v1"], - count: 28, - }, - "openai-responses": { bases: ["https://opencode.ai/zen/v1"], count: 30 }, - }); - // F10: no Google model on Go. - expect(byProtocol(profileOf(tier, "opencode-go").piProvider)).toEqual({ - "anthropic-messages": { bases: ["https://opencode.ai/zen/go"], count: 5 }, - "openai-completions": { - bases: ["https://opencode.ai/zen/go/v1"], - count: 19, - }, - "openai-responses": { - bases: ["https://opencode.ai/zen/go/v1"], - count: 6, - }, - }); + expect(byProtocol(zen.piProvider)).toEqual(forHost(WHOLE_CATALOG).zen); + expect(byProtocol(profileOf(tier, "opencode-go").piProvider)).toEqual( + forHost(WHOLE_CATALOG).go + ); }); it("F5 — a route-level api and baseURL override every model, collapsing pi's four protocols onto one (llm-pi-ai/src/catalog.ts:917,922)", async () => { @@ -387,20 +495,26 @@ describe("host · ResolvedPiAiProviderProfile (llm-pi-ai/src/config.ts:186-219,4 }); const zen = profileOf(tier, "opencode"); expect(zen.catalogError).toBeUndefined(); - // Every one of F2's 83 Zen models now speaks Responses; each keeps its own + // Every one of F2's Zen models now speaks Responses; each keeps its own // base, which only the route's baseURL would override. - expect(byProtocol(zen.piProvider)).toEqual({ + const whole = forHost(WHOLE_CATALOG); + expect(byProtocol(zen.piProvider)).toEqual( + collapsed(whole.zen, "openai-responses") + ); + // Go's models keep their protocols and all move to the route's base. + expect(byProtocol(profileOf(tier, "opencode-go").piProvider)).toEqual( + rebased(whole.go, "https://gateway.test/go") + ); + // A known answer for the derivation: pi 1.1.0's 83 Zen models, on two + // bases, measured before the tables existed. + expect( + collapsed(WHOLE_CATALOG["rc-floor"].zen, "openai-responses") + ).toEqual({ "openai-responses": { bases: ["https://opencode.ai/zen", "https://opencode.ai/zen/v1"], count: 83, }, }); - // Go's 30 models keep their protocols and all move to the route's base. - expect(byProtocol(profileOf(tier, "opencode-go").piProvider)).toEqual({ - "anthropic-messages": { bases: ["https://gateway.test/go"], count: 5 }, - "openai-completions": { bases: ["https://gateway.test/go"], count: 19 }, - "openai-responses": { bases: ["https://gateway.test/go"], count: 6 }, - }); }); }); @@ -424,7 +538,7 @@ describe("host · the loader's interpolate (vendor/loader/src/config/utils.ts:12 }); }); -describe("host · pi 1.1.0's public exports (package.json exports ./providers/*, ./api/*, ./models)", () => { +describe(`host · pi ${PI}'s public exports (package.json exports ./providers/*, ./api/*, ./models)`, () => { it("opencodeProvider/opencodeGoProvider resolve keys through pi's ambient auth (providers/opencode.js:10-24; opencode-go.js:8-20)", async () => { const { opencodeProvider } = await import("@earendil-works/pi-ai/providers/opencode"); @@ -462,41 +576,74 @@ describe("host · pi 1.1.0's public exports (package.json exports ./providers/*, } }); - it("F10 — the Go provider has no Google API: a Google model streams pi's own 'no API implementation' error", async () => { - const { opencodeProvider } = - await import("@earendil-works/pi-ai/providers/opencode"); - const { opencodeGoProvider } = - await import("@earendil-works/pi-ai/providers/opencode-go"); - const apisOf = (provider: { getModels: () => readonly PiModel[] }) => - [...new Set(provider.getModels().map((model) => model.api))].toSorted(); - expect(apisOf(opencodeProvider())).toEqual([ - "anthropic-messages", - "google-generative-ai", - "openai-completions", - "openai-responses", - ]); - expect(apisOf(opencodeGoProvider())).toEqual([ - "anthropic-messages", - "openai-completions", - "openai-responses", - ]); - const gemini = opencodeProvider() - .getModels() - .find((model) => model.api === "google-generative-ai"); - expect(gemini).toBeDefined(); - const events: unknown[] = []; - for await (const event of opencodeGoProvider().stream( - { ...gemini, provider: "opencode-go" } as never, - { messages: [] } as never, - {} - )) { - events.push(event); - } - expect(JSON.stringify(events)).toContain( - 'Provider opencode-go has no API implementation for \\"google-generative-ai\\"' - ); + /** + * F10, per cell: pi 1.1.0's Go provider has no Google API; pi 1.3.0 added + * it and 1.5.0 keeps it. A fact about pi, not a limitation of the plugin: + * the gateway's Go plane serves Google models whichever pi runs, and the + * plugin routes them itself (the Google bridge today, the plane adapter + * from P-R1), so neither cell's row is a `HOST_LIMITATION`. + */ + const GO_HAS_GOOGLE = forHost({ + "alpha-floor": true, + newest: true, + "rc-floor": false, }); + it.runIf(GO_HAS_GOOGLE)( + `[BASELINE] F10 — pi ${PI}'s Go provider carries the Google API (pi 1.3.0 added it)`, + async () => { + const { opencodeGoProvider } = + await import("@earendil-works/pi-ai/providers/opencode-go"); + const models = opencodeGoProvider().getModels() as readonly PiModel[]; + expect([...new Set(models.map((model) => model.api))].toSorted()).toEqual( + [ + "anthropic-messages", + "google-generative-ai", + "openai-completions", + "openai-responses", + ] + ); + } + ); + + it.skipIf(GO_HAS_GOOGLE)( + `[BASELINE] F10 — pi ${PI}'s Go provider has no Google API: a Google model streams pi's own 'no API implementation' error`, + async () => { + const { opencodeProvider } = + await import("@earendil-works/pi-ai/providers/opencode"); + const { opencodeGoProvider } = + await import("@earendil-works/pi-ai/providers/opencode-go"); + const apisOf = (provider: { getModels: () => readonly PiModel[] }) => + [...new Set(provider.getModels().map((model) => model.api))].toSorted(); + expect(apisOf(opencodeProvider())).toEqual([ + "anthropic-messages", + "google-generative-ai", + "openai-completions", + "openai-responses", + ]); + expect(apisOf(opencodeGoProvider())).toEqual([ + "anthropic-messages", + "openai-completions", + "openai-responses", + ]); + const gemini = opencodeProvider() + .getModels() + .find((model) => model.api === "google-generative-ai"); + expect(gemini).toBeDefined(); + const events: unknown[] = []; + for await (const event of opencodeGoProvider().stream( + { ...gemini, provider: "opencode-go" } as never, + { messages: [] } as never, + {} + )) { + events.push(event); + } + expect(JSON.stringify(events)).toContain( + 'Provider opencode-go has no API implementation for \\"google-generative-ai\\"' + ); + } + ); + it("exports createProvider and withOpenCodeSessionHeader (models.js:527; providers/opencode-headers.js:15)", async () => { const { createProvider } = await import("@earendil-works/pi-ai/models"); expect(typeof createProvider).toBe("function"); diff --git a/test/host-waterfall.test.ts b/test/host-waterfall.test.ts index c2440720..61adfe74 100644 --- a/test/host-waterfall.test.ts +++ b/test/host-waterfall.test.ts @@ -140,8 +140,8 @@ describe("host · the internal/config waterfall (tier A)", () => { const input = { providers: s0Providers() }; tier.piFiber.update(input); await tier.piFiber.await(); - // Two runs: `update` resolves while ACTIVE (fiber.ts:747), then the - // restart re-resolves the stored raw (fiber.ts:655). B brackets A both + // Two runs: `update` resolves while ACTIVE (fiber.ts:747 on rc, 776 on alpha), then the + // restart re-resolves the stored raw (fiber.ts:655 on rc, 682 on alpha). B brackets A both // times, so B is outermost and A runs inside the chain B started. expect(order).toEqual(["B:enter", "A", "B:exit", "B:enter", "A", "B:exit"]); for (const record of seen) { @@ -212,7 +212,7 @@ describe("host · the internal/config waterfall (tier A)", () => { expect(order).toEqual(["B:enter", "A", "B:pi-ai threw"]); }); - it("F14 — raw === fiber._config on _reload and on update (cordis fiber.ts:641-644,655,738,747)", async () => { + it("F14 — raw === fiber._config on _reload and on update (cordis fiber.ts: rc 641-644,655,738,747; alpha 667-670,682,767,776)", async () => { tier = await mountTierA({ providers: s0Providers() }); const { piFiber } = tier; const spy = waterfallSpy(tier.ctx, () => piFiber); @@ -232,7 +232,7 @@ describe("host · the internal/config waterfall (tier A)", () => { expect(rawOf(piFiber)).toBe(next); }); - it("F12 — a deep-frozen input is never mutated, and the fiber keeps the outermost listener's return value (cordis fiber.ts:642,655; utils.ts:117-125)", async () => { + it("F12 — a deep-frozen input is never mutated, and the fiber keeps the outermost listener's return value (cordis fiber.ts: rc 642,655; alpha 669,682; utils.ts:117-125)", async () => { const frozen = deepFreeze({ providers: s0Providers() }); tier = await mountTierA({ before: (ctx) => { diff --git a/test/keyless-matrix.test.ts b/test/keyless-matrix.test.ts index f616e059..d4b5c136 100644 --- a/test/keyless-matrix.test.ts +++ b/test/keyless-matrix.test.ts @@ -12,7 +12,7 @@ */ import { describe, expect, it } from "vitest"; -import { PLANES, S0_TODAY } from "./support/host.ts"; +import { HOST_CELLS, PLANES, S0_TODAY_BY_CELL } from "./support/host.ts"; import { cellKey, classifyKeylessProbe, @@ -278,12 +278,15 @@ describe("keyless matrix · the cells", () => { }); describe("keyless matrix · the cells the live run asserts", () => { - it("TODAY_DISPATCH is exactly the cells S0_TODAY's measured turns were sent to", () => { - // Each URL a characterized turn sent is matched to the one cell whose PATH - // it is (the query aside: Messages adds `?beta=true`), with the row's own - // model in the Google path. A row that sent nothing adds nothing. + /** + * The matrix cells one host cell's characterized turns were sent to, and + * the URLs that match no cell (the root family, `/inference(/go)/v1`, which + * the matrix does not probe yet: P-R2 adds it). + */ + const reachedOn = (cell: (typeof HOST_CELLS)[number]) => { const reached = new Set(); - for (const [rowKey, row] of Object.entries(S0_TODAY.rows)) { + const unmatched: string[] = []; + for (const [rowKey, row] of Object.entries(S0_TODAY_BY_CELL[cell].rows)) { const slash = rowKey.indexOf("/"); const plane = rowKey.slice(0, slash) as (typeof PLANES)[number]; const model = rowKey.slice(slash + 1); @@ -298,13 +301,51 @@ describe("keyless matrix · the cells the live run asserts", () => { ) .map((family) => cellKey({ family, plane, protocol })) ); - expect(matches, `${rowKey} ${sent}`).toHaveLength(1); - reached.add(matches[0] ?? ""); + expect(matches.length, `${cell} ${rowKey} ${sent}`).toBeLessThan(2); + if (matches[0] === undefined) { + unmatched.push(new URL(sent).pathname); + } else { + reached.add(matches[0]); + } } } - expect([...reached].toSorted()).toEqual([...TODAY_DISPATCH].toSorted()); - // Completions reach the legacy family today: the modern completions cells - // are P-R1's target, asserted live as a forward guard, never as today's. + return { reached: [...reached].toSorted(), unmatched }; + }; + + it("TODAY_DISPATCH is exactly the cells the rc floor's measured turns were sent to, and every other host cell's are TODAY_DISPATCH with completions moved by its pi", () => { + // Each URL a characterized turn sent is matched to the one cell whose PATH + // it is (the query aside: Messages adds `?beta=true`), with the row's own + // model in the Google path. A row that sent nothing adds nothing. The + // plugin's own routes (Responses, Messages, Google) are the same on every + // host cell; completions follow pi's catalog base, which moved twice. + const floor = reachedOn("rc-floor"); + expect(floor.unmatched).toEqual([]); + const { reached } = floor; + const hidden = [...TODAY_DISPATCH].filter( + (cell) => !cell.includes("openai-completions") + ); + // pi 1.5.0 (newest) files completions under the modern family: exactly + // the forward-guard cells P-R1 targets. + const newest = reachedOn("newest"); + expect(newest.unmatched).toEqual([]); + expect(newest.reached).toEqual([...hidden, ...NEXT_DISPATCH].toSorted()); + // pi 1.3.0 (alpha's floor) files them under the root family, which no + // matrix cell names yet. + const alphaFloor = reachedOn("alpha-floor"); + expect(alphaFloor.reached).toEqual(hidden.toSorted()); + expect(alphaFloor.unmatched.toSorted()).toEqual([ + "/inference/go/v1/chat/completions", + "/inference/v1/chat/completions", + ]); + expect(HOST_CELLS.toSorted()).toEqual([ + "alpha-floor", + "newest", + "rc-floor", + ]); + expect(reached).toEqual([...TODAY_DISPATCH].toSorted()); + // On the rc floor completions reach the legacy family; the modern + // completions cells are where pi 1.5.0 sends them already and P-R1 sends + // them on every host, so the live run asserts both sets. expect([...TODAY_DISPATCH]).toContain("opencode openai-completions legacy"); expect( [...NEXT_DISPATCH].filter((cell) => TODAY_DISPATCH.has(cell)) diff --git a/test/support/host.ts b/test/support/host.ts index 77222897..5f2b145a 100644 --- a/test/support/host.ts +++ b/test/support/host.ts @@ -1,9 +1,12 @@ /** * The real host, in-process: the harness the routing work is judged against. * - * Two tiers, both built from the host packages this repository installs at the - * versions `host-versions.ts` names (pi 1.1.0 patched, the `@deepseek-ai` - * cluster at the published host's tag): + * Two tiers, both built from the host packages this repository installs: the + * harness channel `host-versions.ts` selects (`rc` or `alpha`, each with its + * `@deepseek-ai` cluster at the published host's tag), with pi UNPATCHED at + * the newest version the bridge's range admits, or at the range's floor after + * `scripts/use-channel.ts --floor`. Where the hosts differ, the rows + * below carry one expectation per host cell ({@link HostCell}, {@link forHost}): * * - **Tier A** ({@link mountTierA}): a bare cordis `Context`, `LlmRuntime`, and * `dsh-llm-pi-ai` mounted as a plugin with a `{ providers }` config, so its @@ -89,6 +92,7 @@ import { LlmRuntime } from "@deepseek-ai/dsh-llm"; import * as LlmPiAi from "@deepseek-ai/dsh-llm-pi-ai"; import Settings from "@deepseek-ai/dsh-settings"; +import { type Channel, CHANNELS, HOST_VERSIONS } from "./host-versions.ts"; import { apply, Config, inject, name } from "./src.ts"; /** The package name the plugin looks pi-ai up by (`PI_AI_PACKAGE` in `src/responses-provider.ts`). */ @@ -404,7 +408,8 @@ export const recordingFetch = ( /** * The profile run S0 measured: on each plane one known model per protocol plus - * the models pi 1.1.0 lacks, and a sibling pi-ai route that is not a plane. + * the models pi 1.1.0 lacks (the rc channel's floor; a newer pi describes the + * Go ones, see {@link PI_GAPS}), and a sibling pi-ai route that is not a plane. * `deepseek`'s base is loopback port 9 (discard), so nothing reaches it. */ export const S0_PROFILE = { @@ -435,7 +440,7 @@ export const S0_PROFILE = { } as const; /** - * A DECLARED id that neither pi 1.1.0 nor the plugin's catalog describes (G1's + * A DECLARED id that no supported pi nor the plugin's catalog describes (G1's * second class, beside the gap ids the plugin's catalog still describes). */ export const UNKNOWN_ID = "not-in-any-catalog"; @@ -492,8 +497,110 @@ export const CHARACTERIZED_MODELS: Readonly> = * How a characterization row is tagged, so a later phase flips named rows: * `KNOWN_DEFECT` is wrong today and a phase fixes it, `INTENDED_CHANGE` is * right today and a phase changes it on purpose, `BASELINE` must survive. + * `HOST_LIMITATION` is something one host cell's pi lacks that the vendor's + * service needs: no phase of this plugin fixes it (D26: a host gap is the + * host's job), its title names the channel and the pi it applies to, and it + * is reported upstream. */ -export type RowTag = "BASELINE" | "INTENDED_CHANGE" | "KNOWN_DEFECT"; +export type RowTag = + | "BASELINE" + | "HOST_LIMITATION" + | "INTENDED_CHANGE" + | "KNOWN_DEFECT"; + +// --------------------------------------------------------------------------- +// The host cells +// --------------------------------------------------------------------------- + +/** + * The installs a row is measured on (D14''). `newest` is either channel's + * default install: pi UNPATCHED at the newest version the bridge's range + * admits, what an npm user of the channel runs. Both channels resolve the same + * pi today, so they share the column; a fixture case fails the day the two + * records' `pi` differ, and the column is split then rather than silently + * shared. `rc-floor` and `alpha-floor` are each channel's range floor, the pi + * the harness's own workspace pins and the desktop app ships (patched there, + * not here: the patch touches nothing the plugin reads). + */ +export type HostCell = "alpha-floor" | "newest" | "rc-floor"; + +/** + * Every {@link HostCell}, built from a `Record` keyed by the union so a cell + * added to the type and missing here fails to compile. + */ +export const HOST_CELLS = Object.keys({ + "alpha-floor": true, + newest: true, + "rc-floor": true, +} satisfies Record) as readonly HostCell[]; + +/** + * The cell an install is, from its record's channel and floor flag. + * + * @param host - the selected host (`HOST_VERSIONS`, or a probe's record). + * @returns the cell. + */ +export const hostCellOf = (host: { + readonly channel: Channel; + readonly floor: boolean; +}): HostCell => (host.floor ? `${host.channel}-floor` : "newest"); + +/** The cell the install under test is: `HOST_VERSIONS`, which `DSH_CHANNEL`/`DSH_PI` select. */ +export const HOST_CELL: HostCell = hostCellOf(HOST_VERSIONS); + +/** The pi each cell runs, read from the channel records rather than restated. */ +export const PI_OF_CELL: Readonly> = { + "alpha-floor": CHANNELS.alpha.piFloor, + newest: CHANNELS.alpha.pi, + "rc-floor": CHANNELS.rc.piFloor, +}; + +/** + * One expectation per host cell, picked for the install under test. Keyed by + * the cell's ROLE, never by a pi version, so a bump that moves `newest` keeps + * its rows and fails only where the new pi behaves differently. + * + * @param values - the expectation for each cell. + * @param cell - the cell to read, the install under test's by default. + * @returns that cell's expectation. + */ +export const forHost = ( + values: Readonly>, + cell: HostCell = HOST_CELL +): T => values[cell]; + +/** + * One expectation per channel, for what the `@deepseek-ai` cluster decides + * rather than pi (pi-ai's own defaults, say). + * + * @param values - the expectation for each channel. + * @returns the selected channel's. + */ +export const forChannel = (values: Readonly>): T => + values[HOST_VERSIONS.channel]; + +/** Which declared S0 ids each plane's pi cannot place, per cell. */ +type PlaneIds = Readonly>; + +/** + * The declared S0 ids each cell's pi cannot place (its catalog does not + * describe them, while the plugin's catalog still does), in declaration + * order. pi 1.5.0 describes the two Go ones; no supported pi describes Zen's. + */ +export const PI_GAPS_BY_CELL: Readonly> = { + "alpha-floor": { + opencode: ["gpt-5.3-codex-spark"], + "opencode-go": ["gemini-4-argon", "minimax-m3.1-flash"], + }, + newest: { opencode: ["gpt-5.3-codex-spark"], "opencode-go": [] }, + "rc-floor": { + opencode: ["gpt-5.3-codex-spark"], + "opencode-go": ["gemini-4-argon", "minimax-m3.1-flash"], + }, +}; + +/** {@link PI_GAPS_BY_CELL} for the install under test. */ +export const PI_GAPS: PlaneIds = forHost(PI_GAPS_BY_CELL); /** * The three facets a row records, each tagged on its own: one model can be @@ -503,7 +610,7 @@ export type RowTag = "BASELINE" | "INTENDED_CHANGE" | "KNOWN_DEFECT"; */ export type RowFacet = "callConfig" | "info" | "wire"; -/** What TODAY's plugin does with one characterized model on pi 1.1.0, as measured. */ +/** What TODAY's plugin does with one characterized model on one host cell, as measured. */ export interface TodayRow { readonly tags: Readonly>; /** @@ -536,10 +643,12 @@ const I = "INTENDED_CHANGE"; /** * TODAY's plugin (hidden routes, listing patch, adapter decoration and all) on - * pi 1.1.0 with {@link CHARACTERIZED_PROFILE} and a key stored for each plane, - * measured in tier B with the published host's apply order (`plugin-first`, - * see {@link ApplyOrder}) and shared with the binary e2e so the two cannot - * drift. Each facet carries its own tag, and the characterization generates + * the `rc-floor` cell (pi 1.1.0) with {@link CHARACTERIZED_PROFILE} and a key + * stored for each plane, measured in tier B with the published host's apply + * order (`plugin-first`, see {@link ApplyOrder}). The other cells are this + * table with the facets their pi moves ({@link S0_TODAY_BY_CELL}), and + * {@link S0_TODAY} is the install under test's, shared with the binary e2e so + * the two cannot drift. Each facet carries its own tag, and the characterization generates * one test per row and facet titled with it: P-R1 flips the `KNOWN_DEFECT` and * `INTENDED_CHANGE` facets its PR body names, and no `BASELINE` facet moves — * except that P-R2's measured family flip, if a `modern` cell fails, may move @@ -557,7 +666,7 @@ const I = "INTENDED_CHANGE"; * reads the same table, but its job runs on a PR only when this file is in * its path scope (`.github/workflows/ci.yml`, `e2e-host`), which it is. */ -export const S0_TODAY = { +const S0_RC_FLOOR = { /** `listModels` before any turn: the plugin sees no key, so it hides paid models. */ keylessListing: { opencode: [], "opencode-go": [] }, /** `listModels` once a turn captured a key: pi-resolved ids only. */ @@ -692,20 +801,121 @@ export const S0_TODAY = { /provider "opencode" model "not-a-real-model" needs an api/u, } as const; +/** A characterized row's key, `/`. */ +export type S0RowKey = keyof typeof S0_RC_FLOOR.rows; + +/** One cell's characterization table. */ +export interface S0Table { + readonly keylessListing: PlaneIds; + readonly keyedListing: PlaneIds; + readonly rows: Readonly>; + readonly saveBlockedByStoredGap: RegExp; + readonly saveRejectedUnknown: RegExp; +} + +/** + * A cell's table: `base` with the rows `moved` replaces, each replacement a + * whole row so its tags are restated with its values. + */ +const withMoved = ( + base: S0Table, + moved: Partial>, + listings: Partial> = {} +): S0Table => ({ + ...base, + ...listings, + rows: { ...base.rows, ...moved }, +}); + +/** + * pi 1.3.0 (alpha's floor) files its catalog's completions models under the + * root family, `/inference(/go)/v1`: the turn leaves the legacy base, but not + * for the per-protocol one P-R1 moves it to, so the facet stays + * `INTENDED_CHANGE`. It still lacks both Go gap ids. + */ +const S0_ALPHA_FLOOR = withMoved(S0_RC_FLOOR, { + "opencode-go/deepseek-v4-pro": { + ...S0_RC_FLOOR.rows["opencode-go/deepseek-v4-pro"], + sent: [`${ZEN}/inference/go/v1/chat/completions`], + }, + "opencode/deepseek-v4-pro": { + ...S0_RC_FLOOR.rows["opencode/deepseek-v4-pro"], + sent: [`${ZEN}/inference/v1/chat/completions`], + }, +}); + +/** + * pi 1.5.0 (both channels' newest) files completions under the per-protocol + * family P-R1 targets, and describes both Go gap ids, so each facet that + * moves here is already what P-R1 keeps and is `BASELINE` on this cell: + * completions reach `/inference(/go)/openai/v1`, Go gemini-4-argon validates + * in either apply order, and Go minimax-m3.1-flash is placed (on + * completions, which is the plugin catalog's SDK too) and sent. + */ +const S0_NEWEST = withMoved( + S0_RC_FLOOR, + { + "opencode-go/deepseek-v4-pro": { + ...S0_RC_FLOOR.rows["opencode-go/deepseek-v4-pro"], + sent: [`${ZEN}/inference/go/openai/v1/chat/completions`], + tags: tags(B, B, B), + }, + "opencode-go/gemini-4-argon": { + ...S0_RC_FLOOR.rows["opencode-go/gemini-4-argon"], + callConfig: "ok", + tags: tags(B, B, B), + }, + "opencode-go/minimax-m3.1-flash": { + ...S0_RC_FLOOR.rows["opencode-go/minimax-m3.1-flash"], + callConfig: "ok", + finish: "AUTH", + sent: [`${ZEN}/inference/go/openai/v1/chat/completions`], + tags: tags(B, B, B), + }, + "opencode/deepseek-v4-pro": { + ...S0_RC_FLOOR.rows["opencode/deepseek-v4-pro"], + sent: [`${ZEN}/inference/openai/v1/chat/completions`], + tags: tags(B, B, B), + }, + }, + { + keyedListing: { + ...S0_RC_FLOOR.keyedListing, + "opencode-go": [ + ...S0_RC_FLOOR.keyedListing["opencode-go"], + "gemini-4-argon", + "minimax-m3.1-flash", + ], + }, + } +); + +/** Every cell's table, for a case that checks them all. */ +export const S0_TODAY_BY_CELL: Readonly> = { + "alpha-floor": S0_ALPHA_FLOOR, + newest: S0_NEWEST, + "rc-floor": S0_RC_FLOOR, +}; + +/** The install under test's table ({@link HOST_CELL}). */ +export const S0_TODAY: S0Table = forHost(S0_TODAY_BY_CELL); + /** The id the save rows try to add: unknown to every catalog, and not declared. */ export const SAVE_CANDIDATE = "not-a-real-model"; /** - * The ids on `opencode` that pi 1.1.0 cannot place: the stored gap id and the + * The ids on `opencode` that no supported pi can place: the stored gap id and the * declared unknown id. The save sequence (in-process, and the probe's * `PROBE_SAVE_DROP`) removes them before saving {@link SAVE_CANDIDATE}. */ export const ZEN_UNDESCRIBED = ["gpt-5.3-codex-spark", UNKNOWN_ID] as const; /** - * TODAY's rows for an apply order. `S0_TODAY.rows` is the `plugin-first` - * table; with `pi-ai-first` the adapter decoration wraps pi-ai's plane adapter - * and exactly one facet moves — Go gemini-4-argon validates — + * TODAY's rows for an apply order on a host cell. A cell's `rows` is its + * `plugin-first` table; with `pi-ai-first` the adapter decoration wraps + * pi-ai's plane adapter and on the floor cells exactly one facet moves — Go + * gemini-4-argon validates (on `newest` pi already places it, so nothing + * moves) — * measured in `test/host-characterization.test.ts`, which generates one test * per row and facet for each order, titled with that order's tags. The moved * facet is `BASELINE` in this table: its `"ok"` is already what P-R1 keeps, so @@ -715,13 +925,16 @@ export const ZEN_UNDESCRIBED = ["gpt-5.3-codex-spark", UNKNOWN_ID] as const; * host the order is a race (F17) the suite must not pin. * * @param order - the apply order, as inferred from the decoration. + * @param cell - the host cell, the install under test's by default. * @returns the rows, keyed `/`. */ export const todayRowsFor = ( - order: ApplyOrder -): Readonly> => { + order: ApplyOrder, + cell: HostCell = HOST_CELL +): Readonly> => { + const { rows } = S0_TODAY_BY_CELL[cell]; if (order === "plugin-first") { - return S0_TODAY.rows; + return rows; } // Exhaustive on purpose: a probe JSON with a missing or renamed field reads // `undefined` here, and must not fall through to the pi-ai-first table. @@ -729,12 +942,12 @@ export const todayRowsFor = ( throw new Error(`todayRowsFor: unknown apply order ${String(order)}`); } return { - ...S0_TODAY.rows, + ...rows, "opencode-go/gemini-4-argon": { - ...S0_TODAY.rows["opencode-go/gemini-4-argon"], + ...rows["opencode-go/gemini-4-argon"], callConfig: "ok", tags: { - ...S0_TODAY.rows["opencode-go/gemini-4-argon"].tags, + ...rows["opencode-go/gemini-4-argon"].tags, callConfig: B, }, }, @@ -754,9 +967,10 @@ export const APPLY_ORDERS = Object.keys({ /** * What appending a new id to `opencode` answers while the plane stores ANY of * {@link ZEN_UNDESCRIBED}, whichever one the installed pi cannot describe - * first. The pinned pi always names `gpt-5.3-codex-spark` - * (`S0_TODAY.saveBlockedByStoredGap`); a floating pi that learned it names the - * next one, so the floating cell asserts this instead. + * first. Every recorded cell's pi names `gpt-5.3-codex-spark` + * (`S0_TODAY.saveBlockedByStoredGap`); a binary whose npm-resolved pi moved + * past its channel record may have learned it and name the next one, so the + * binary e2e asserts this instead on such a pi. */ export const SAVE_BLOCKED_BY_ANY_UNDESCRIBED = new RegExp( `provider "opencode" model "(?:${ZEN_UNDESCRIBED.map((id) => @@ -784,7 +998,8 @@ const piData = (plane: Plane): Record> => /** * The protocol the installed pi's catalog files a model under on a plane, or * `undefined` when pi does not know it. The data is keyed by protocol, then - * `chat:`, so a fixture can check which of its ids are gaps on pi 1.1.0. + * `chat:`, so a fixture can check which of its ids are gaps on the + * installed pi ({@link PI_GAPS}). * * @param plane - the plane route. * @param id - the model id. diff --git a/test/support/keyless-matrix.ts b/test/support/keyless-matrix.ts index d83d6235..591bd412 100644 --- a/test/support/keyless-matrix.ts +++ b/test/support/keyless-matrix.ts @@ -57,13 +57,17 @@ export const cellKey = ( /** * The cells TODAY's plugin sends turns to with the default endpoint (`auto`, - * an alias of `modern`), on both planes: completions on pi 1.1.0's LEGACY - * bases (`/zen(/go)/v1/chat/completions`, pi-ai's own plane route) and the - * three hidden routes' MODERN paths (Responses, Messages, Google). A path-gone + * an alias of `modern`), on both planes, on the oldest supported host (the rc + * channel's floor, pi 1.1.0): completions on pi's LEGACY bases + * (`/zen(/go)/v1/chat/completions`, pi-ai's own plane route) and the three + * hidden routes' MODERN paths (Responses, Messages, Google). A path-gone * verdict on one of these breaks a shipped model in the default - * configuration, so the live matrix asserts them. `test/keyless-matrix.test.ts` - * derives the same set from the URLs `S0_TODAY` measured (with the default - * endpoint), so the two cannot drift. + * configuration, so the live matrix asserts them. Completions follow pi's + * catalog: pi 1.5.0, either channel's newest, sends them to the modern cells + * ({@link NEXT_DISPATCH}, also asserted live), and pi 1.3.0, alpha's floor, to + * the root family `/inference(/go)/v1`, which no cell here names yet. + * `test/keyless-matrix.test.ts` derives all three from the URLs each host + * cell's `S0_TODAY` table measured, so they cannot drift. * * Not covered: a user who sets `endpoint: legacy` dispatches Responses and * Messages on both planes, and Zen's Google, to the LEGACY cells, which the From 017adad941222ade11f74565e1cb10d4eb6ae831 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 16:01:07 +0000 Subject: [PATCH 05/11] ci: boot each channel's published binary, as npm installs it and at the floor The e2e-host job installed one host with pi forced to the harness pin and patched. It now runs per channel with pi unpatched: the npm cell on pull requests touching the host seam, the floor cell nightly and on dispatch, each asserting its own cell's rows. A pi newer than the channel record skips only the pi-dependent rows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- .github/workflows/ci.yml | 124 ++++++++++++------------- test/e2e/host-binary.e2e.ts | 179 ++++++++++++++++++++++++------------ 2 files changed, 180 insertions(+), 123 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c7af05e..cecd16cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -392,30 +392,37 @@ jobs: # the unit suite assembles. `test/e2e/host/probe-plugin.mjs` measures the # registry, the listings, validation, where each turn is sent and the save of # a catalog-unknown id, and `test/e2e/host-binary.e2e.ts` asserts today's - # rows (`S0_TODAY`, shared with the in-process characterization). + # rows (`S0_TODAY_BY_CELL`, shared with the in-process characterization). # - # Two cells. `pinned` installs the host with npm `overrides` forcing pi, - # cordis and the loader to the versions `test/support/host-versions.ts` - # names (the published host declares the last two as tilde ranges, so a - # fresh install could otherwise take a newer patch) and applies the - # harness's own patch to pi; `floating` takes whatever pi npm resolves for the - # published host (newer than the harness pins), so a pi release that moves - # the ground shows up nightly. Pull requests run `pinned` only, and only when - # they touch the host seam; dispatch and the nightly run both cells. + # One run per harness channel (D14''), each the channel's published binary + # (`dsh@` from `test/support/host-versions.ts`) installed with npm the + # way a user installs it, pi unpatched in both cells: + # - `npm`: pi as npm resolves it for the published host. The record's + # newest pi asserts every row; a pi published since the record skips the + # rows that depend on which ids pi describes, with a warning (the nightly + # `host-drift` job names it). Never cached: re-resolving is the point. + # - `floor`: pi forced to the bridge range's floor with npm `overrides`, the + # pi the desktop app ships (patched there, not here; the patch touches + # nothing the plugin reads). Cached by channel, dsh and pi. + # Pull requests run the `npm` cell of both channels, and only when they touch + # the host seam; dispatch and the nightly run add both `floor` cells. # # The scope is decided by a step rather than by `on.pull_request.paths`: a # workflow-level path filter would silence every other job in this file. # Offline: the probe answers every gateway request itself, and the keys it # stores are dummies. No secret is passed. e2e-host: - name: e2e · host binary (${{ matrix.cell }}) + name: e2e · host binary (${{ matrix.channel }} · ${{ matrix.cell }}) if: github.event_name != 'push' runs-on: ubuntu-latest timeout-minutes: 25 strategy: fail-fast: false matrix: - cell: ${{ github.event_name == 'pull_request' && fromJSON('["pinned"]') || fromJSON('["pinned","floating"]') }} + channel: [alpha, rc] + cell: ${{ github.event_name == 'pull_request' && fromJSON('["npm"]') || fromJSON('["npm","floor"]') }} + env: + DSH_CHANNEL: ${{ matrix.channel }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -436,15 +443,15 @@ jobs: # than reading as "nothing changed" and skipping the job green. changed="$(git diff --name-only "origin/${BASE_REF}...HEAD")" # The job's own definition and runner config are in scope: a pull - # request that breaks the install, the override, the patch step or - # the e2e config must run the job it breaks. That includes - # vite.config.ts: it holds the pack config that builds lib/index.mjs - # (the only test that loads that bundle into a real host is this - # one, and its externals keep cordis and schemastery the host's - # copies) and the test settings vitest.e2e.config.ts inherits. - # package.json is not (every dependency bump touches it), so a - # change to its `test:e2e:host` script alone is exercised only by - # the nightly run. + # request that breaks the install, the override or the e2e config + # must run the job it breaks. That includes vite.config.ts: it holds + # the pack config that builds lib/index.mjs (the only test that + # loads that bundle into a real host is this one, and its externals + # keep cordis and schemastery the host's copies) and the test + # settings vitest.e2e.config.ts inherits. test/support/host-versions.ts + # names the binary each channel installs. package.json is not (every + # dependency bump touches it), so a change to its `test:e2e:host` + # script alone is exercised only by the nightly run. # Until P-R1 moves the host seam under src/host/, the modules that # install the host patches and mounts whose behaviour depends on the # real loader and apply order (the hidden-route mount, the listing @@ -452,10 +459,10 @@ jobs: # the plane bases and the stream hook) are in scope by name. Every # other row producer (key-capture.ts, models-catalog.ts, config.ts, # the generated catalog) is covered on each pull request by the - # in-process characterization of the same S0_TODAY rows, and here by - # the nightly run. + # in-process characterization of the same rows, on both channels, and + # here by the nightly run. if printf '%s\n' "$changed" \ - | grep -Eq '^(src/host/|src/lifecycle\.ts$|src/identity\.ts$|src/responses-provider\.ts$|src/responses-routes\.ts$|src/models-discovery\.ts$|src/google-bridge\.ts$|src/fetch-patch\.ts$|src/pi-provider\.ts$|src/stream-hook\.ts$|cordis\.patch\.yml$|test/e2e/host/|test/e2e/host-binary\.e2e\.ts$|test/support/host\.ts$|test/support/host-versions\.ts$|patches/|\.github/workflows/ci\.yml$|vitest\.e2e\.config\.ts$|vite\.config\.ts$)'; then + | grep -Eq '^(src/host/|src/lifecycle\.ts$|src/identity\.ts$|src/responses-provider\.ts$|src/responses-routes\.ts$|src/models-discovery\.ts$|src/google-bridge\.ts$|src/fetch-patch\.ts$|src/pi-provider\.ts$|src/stream-hook\.ts$|cordis\.patch\.yml$|test/e2e/host/|test/e2e/host-binary\.e2e\.ts$|test/support/host\.ts$|test/support/host-versions\.ts$|\.github/workflows/ci\.yml$|vitest\.e2e\.config\.ts$|vite\.config\.ts$)'; then echo "run=true" >> "$GITHUB_OUTPUT" else echo "run=false" >> "$GITHUB_OUTPUT" @@ -470,75 +477,65 @@ jobs: cache: pnpm - if: steps.scope.outputs.run == 'true' run: pnpm install --frozen-lockfile - - name: the host versions this repository pins + - name: the ${{ matrix.channel }} channel's record id: versions if: steps.scope.outputs.run == 'true' run: | - node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS } from './test/support/host-versions.ts'; console.log('dsh=' + HOST_VERSIONS.dsh); console.log('pi=' + HOST_VERSIONS.pi); console.log('cordis=' + HOST_VERSIONS.cordis); console.log('loader=' + HOST_VERSIONS.loader)" >> "$GITHUB_OUTPUT" + node --experimental-strip-types --input-type=module -e "import { CHANNELS } from './test/support/host-versions.ts'; const r = CHANNELS[process.env.DSH_CHANNEL]; console.log('dsh=' + r.dsh); console.log('pi=' + r.pi); console.log('piFloor=' + r.piFloor); console.log('piRange=' + r.piRange)" >> "$GITHUB_OUTPUT" echo "PREFIX=$RUNNER_TEMP/dsh-host" >> "$GITHUB_ENV" - # The pinned install forces every host version HOST_VERSIONS names, so it - # is cached by those versions and the patch (the rest of the published - # host's tree is whatever its ranges resolved on the miss that filled - # the cache); the floating one is the point of re-resolving, so never. - - name: cache the pinned host install + # The floor install forces one version, so it is cached by the channel, + # its dsh and that pi (the rest of the published host's tree is whatever + # its ranges resolved on the miss that filled the cache); the npm one is + # the point of re-resolving, so never. + - name: cache the floor host install id: host-cache - if: steps.scope.outputs.run == 'true' && matrix.cell == 'pinned' + if: steps.scope.outputs.run == 'true' && matrix.cell == 'floor' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/dsh-host - key: dsh-host-${{ matrix.cell }}-${{ steps.versions.outputs.dsh }}-${{ steps.versions.outputs.pi }}-${{ steps.versions.outputs.cordis }}-${{ steps.versions.outputs.loader }}-${{ hashFiles('patches/*.patch') }} + key: dsh-host-floor-${{ matrix.channel }}-${{ steps.versions.outputs.dsh }}-${{ steps.versions.outputs.piFloor }} - name: install the published host into a throwaway prefix if: steps.scope.outputs.run == 'true' && steps.host-cache.outputs.cache-hit != 'true' env: CELL: ${{ matrix.cell }} DSH: ${{ steps.versions.outputs.dsh }} - PI: ${{ steps.versions.outputs.pi }} - CORDIS: ${{ steps.versions.outputs.cordis }} - LOADER: ${{ steps.versions.outputs.loader }} + PI_FLOOR: ${{ steps.versions.outputs.piFloor }} run: | mkdir -p "$PREFIX" - # The overrides must be in the prefix's own package.json BEFORE the - # install, or npm resolves the wrapper's ^range to the newest pi. - if [ "$CELL" = pinned ]; then - printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s", "@deepseek-ai/cordis": "%s", "@deepseek-ai/cordis-plugin-loader": "%s" } }\n' "$PI" "$CORDIS" "$LOADER" > "$PREFIX/package.json" + # The override must be in the prefix's own package.json BEFORE the + # install, or npm resolves the bridge's ^range to the newest pi. + if [ "$CELL" = floor ]; then + printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s" } }\n' "$PI_FLOOR" > "$PREFIX/package.json" else printf '{ "private": true }\n' > "$PREFIX/package.json" fi npm install --prefix "$PREFIX" --no-audit --no-fund "@deepseek-ai/dsh@$DSH" - # Only on a fresh install: re-applying to a cached, patched copy fails. - if [ "$CELL" = pinned ]; then - patch -p1 -d "$PREFIX/node_modules/@earendil-works/pi-ai" < "patches/@earendil-works__pi-ai@$PI.patch" - fi - - name: assert the pi, cordis and loader the host resolved + - name: report the pi, cordis and loader the host resolved if: steps.scope.outputs.run == 'true' env: CELL: ${{ matrix.cell }} PI: ${{ steps.versions.outputs.pi }} - CORDIS: ${{ steps.versions.outputs.cordis }} - LOADER: ${{ steps.versions.outputs.loader }} + PI_FLOOR: ${{ steps.versions.outputs.piFloor }} run: | resolved="$(node -p "require(process.env.PREFIX + '/node_modules/@earendil-works/pi-ai/package.json').version")" - echo "cell $CELL: pi $resolved" + echo "${DSH_CHANNEL} · ${CELL}: pi $resolved" copies="$(find "$PREFIX/node_modules" -path '*/@earendil-works/pi-ai/package.json' | wc -l)" if [ "$copies" -ne 1 ]; then echo "::error::expected one copy of pi under the host prefix, found $copies" exit 1 fi - if [ "$CELL" = pinned ] && [ "$resolved" != "$PI" ]; then - echo "::error::the pinned cell resolved pi $resolved, not $PI" + if [ "$CELL" = floor ] && [ "$resolved" != "$PI_FLOOR" ]; then + echo "::error::the floor cell resolved pi $resolved, not $PI_FLOOR" exit 1 fi - # The check test/host-versions.test.ts runs for the unit install, - # here for the host the binary e2e boots (a cached prefix included). - for pair in "cordis:$CORDIS" "cordis-plugin-loader:$LOADER"; do - name="@deepseek-ai/${pair%%:*}" - want="${pair#*:}" - got="$(node -p "require(process.env.PREFIX + '/node_modules/$name/package.json').version")" - echo "cell $CELL: $name $got" - if [ "$CELL" = pinned ] && [ "$got" != "$want" ]; then - echo "::error::the pinned cell resolved $name $got, not $want" - exit 1 - fi + if [ "$CELL" = npm ] && [ "$resolved" != "$PI" ]; then + echo "::warning::npm resolved pi $resolved for ${DSH_CHANNEL}, not the record's $PI: the rows that depend on which ids pi describes are skipped" + fi + # Reported, not asserted: the published host declares cordis and the + # loader as tilde ranges, and a new patch there is the channel's + # release, which the nightly drift report names. + for name in @deepseek-ai/cordis @deepseek-ai/cordis-plugin-loader; do + echo "${DSH_CHANNEL} · ${CELL}: $name $(node -p "require(process.env.PREFIX + '/node_modules/$name/package.json').version")" done - if: steps.scope.outputs.run == 'true' run: pnpm run build @@ -549,12 +546,11 @@ jobs: DSH_HOST_E2E: "1" DSH_HOST_PREFIX: ${{ runner.temp }}/dsh-host DSH_HOST_PI_CELL: ${{ matrix.cell }} - DSH_HOST_PROBE_COPY: ${{ runner.temp }}/probe-${{ matrix.cell }}.json + DSH_HOST_PROBE_COPY: ${{ runner.temp }}/probe-${{ matrix.channel }}-${{ matrix.cell }}.json - name: upload the probe's measurement if: always() && steps.scope.outputs.run == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: host-probe-${{ matrix.cell }} - path: ${{ runner.temp }}/probe-${{ matrix.cell }}.json + name: host-probe-${{ matrix.channel }}-${{ matrix.cell }} + path: ${{ runner.temp }}/probe-${{ matrix.channel }}-${{ matrix.cell }}.json if-no-files-found: ignore - retention-days: 7 diff --git a/test/e2e/host-binary.e2e.ts b/test/e2e/host-binary.e2e.ts index e88041d8..9625fcc4 100644 --- a/test/e2e/host-binary.e2e.ts +++ b/test/e2e/host-binary.e2e.ts @@ -20,15 +20,23 @@ * than pinning a 70 ms margin. * * Gated: it runs only with `DSH_HOST_E2E=1`, and needs `DSH_HOST_PREFIX`, a - * directory whose `node_modules/@deepseek-ai/dsh` is the installed host. - * `DSH_HOST_PI_CELL=pinned` (the default) also requires the pi - * `host-versions.ts` names, with the harness patch, and asserts every row; - * `floating` accepts whatever pi the host resolved. It skips the rows that - * depend on which ids pi describes (the keyed listing and the per-model rows) - * and still asserts the save rows. Those assume the floating pi lacks at least - * one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so - * a pi that learns all of them or retires a declared id turns them red by - * design — triage a red floating run as a pi change first. + * directory whose `node_modules/@deepseek-ai/dsh` is the installed host of the + * channel `DSH_CHANNEL` names (`alpha` by default; `host-versions.ts` holds + * both channels' records). The installed dsh must be that record's. + * `DSH_HOST_PI_CELL` says which pi the prefix holds, both unpatched (D14''): + * + * - `npm` (the default): pi as npm resolved it for the published host, what an + * npm user of the channel runs. Its rows are the `newest` cell's while the + * resolved pi is the channel record's `pi`. A pi published since the record + * was cut is still asserted to sit in the bridge's range, but the rows that + * depend on which ids pi describes (the keyed listing and the per-model + * rows) are skipped with a warning: the nightly drift report names the new + * pi, and its probe JSON is the record. The save rows still run; they assume + * pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other + * declared Zen id, so a pi that learns all of them or retires a declared id + * turns them red by design — triage that as a pi change first. + * - `floor`: pi forced to the range's floor, the pi the desktop app ships. The + * resolved pi must be that floor, and every row is the channel's floor cell. * * `DSH_HOST_PROBE_COPY` names a file to keep the probe's JSON in, for the CI * artifact; it is written whenever the probe wrote one, a failed run included. @@ -55,13 +63,14 @@ import { fileURLToPath } from "node:url"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { HOST_VERSIONS } from "../support/host-versions.ts"; +import { CHANNELS, HOST_VERSIONS } from "../support/host-versions.ts"; import { type ApplyOrder, CHARACTERIZED_MODELS, CHARACTERIZED_PROFILE, + type HostCell, PI_AI_PACKAGE, - S0_TODAY, + S0_TODAY_BY_CELL, SAVE_BLOCKED_BY_ANY_UNDESCRIBED, SAVE_CANDIDATE, todayRowsFor, @@ -71,8 +80,41 @@ import { INTERNAL_ROUTES } from "../support/src.ts"; const ENABLED = process.env.DSH_HOST_E2E === "1"; const PREFIX = process.env.DSH_HOST_PREFIX ?? ""; -const CELL = process.env.DSH_HOST_PI_CELL ?? "pinned"; -const PINNED = CELL === "pinned"; +const CELL = process.env.DSH_HOST_PI_CELL ?? "npm"; +if (CELL !== "npm" && CELL !== "floor") { + throw new Error(`DSH_HOST_PI_CELL=${CELL} names no cell; use npm or floor`); +} +/** The channel the prefix holds (`DSH_CHANNEL`, read by `host-versions.ts`). */ +const CHANNEL = HOST_VERSIONS.channel; +const RECORD = CHANNELS[CHANNEL]; +/** The host cell whose rows this run asserts. */ +const ROWS_CELL: HostCell = CELL === "floor" ? `${CHANNEL}-floor` : "newest"; +const ROWS = S0_TODAY_BY_CELL[ROWS_CELL]; +/** The pi that cell's rows were measured on. */ +const ROWS_PI = CELL === "floor" ? RECORD.piFloor : RECORD.pi; +/** + * Whether the prefix's pi is the one the rows were measured on; set in + * `beforeAll`, false for an npm-resolved pi newer than the channel record. + */ +let rowsHold = false; + +/** + * Whether `version` satisfies a caret range `^x.y.z` (x >= 1): the same + * major, and not below the floor. The bridge declares nothing else. + */ +const satisfiesCaret = (version: string, range: string): boolean => { + const parse = (text: string): number[] => + text.replace(/^\^/u, "").split(".").map(Number); + const [major, minor, patch] = parse(version); + const [floorMajor, floorMinor, floorPatch] = parse(range); + if (major !== floorMajor) { + return false; + } + return ( + (minor ?? 0) * 1e6 + (patch ?? 0) >= + (floorMinor ?? 0) * 1e6 + (floorPatch ?? 0) + ); +}; const BOOT_DEADLINE_MS = 180_000; /** Each `dsh plugin install` (a blocking `spawnSync`). */ const INSTALL_DEADLINE_MS = 120_000; @@ -239,14 +281,33 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { version: string; } ).version; - console.info(`[e2e-host] cell ${CELL}: pi ${piVersion}`); + const dshVersion = ( + JSON.parse( + readFileSync( + join(PREFIX, "node_modules", "@deepseek-ai", "dsh", "package.json"), + "utf8" + ) + ) as { version: string } + ).version; + console.info( + `[e2e-host] ${CHANNEL} · ${CELL}: dsh ${dshVersion}, pi ${piVersion} (rows: ${ROWS_CELL}, measured on pi ${ROWS_PI})` + ); + expect(dshVersion, "the prefix holds another channel's host").toBe( + RECORD.dsh + ); expect(piCopies(join(PREFIX, "node_modules"))).toHaveLength(1); - if (PINNED) { - expect(piVersion).toBe(HOST_VERSIONS.pi); - // The harness patch is applied (the check test/host-versions.test.ts runs). - expect( - readFileSync(join(piRoot, "dist", "api", "pi-messages.js"), "utf8") - ).not.toContain("parseStreamingJson(json)"); + expect( + satisfiesCaret(piVersion, RECORD.piRange), + `pi ${piVersion} is outside the bridge's ${RECORD.piRange}` + ).toBe(true); + if (CELL === "floor") { + expect(piVersion).toBe(RECORD.piFloor); + } + rowsHold = piVersion === ROWS_PI; + if (!rowsHold) { + console.warn( + `[e2e-host] npm resolved pi ${piVersion}, not the ${CHANNEL} record's ${RECORD.pi}: the rows that depend on which ids pi describes are skipped (the nightly drift report names it)` + ); } expect(existsSync(join(CHECKOUT, "lib", "index.mjs"))).toBe(true); @@ -428,43 +489,43 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { }); it("[INTENDED_CHANGE] keyless listing: every paid model is hidden before a key is captured", () => { - expect(out.listModels).toEqual(S0_TODAY.keylessListing); + expect(out.listModels).toEqual(ROWS.keylessListing); }); - // These two depend on which ids pi describes, so they are pinned for the - // pi this repository pins; the floating cell skips them and its probe JSON - // is the record. + // These two depend on which ids pi describes, so they hold for the pi the + // cell's rows were measured on; an npm-resolved pi newer than the channel + // record skips them, and its probe JSON is the record. describe("rows that depend on which ids pi describes", () => { - it.skipIf(!PINNED)( - "[KNOWN_DEFECT] keyed listing: the gap ids are missing (pinned pi)", - () => { - expect(out.listModelsAfter).toEqual(S0_TODAY.keyedListing); + it("[KNOWN_DEFECT] keyed listing: the ids this cell's pi cannot place are missing", (context) => { + if (!rowsHold) { + context.skip(); } - ); + expect(out.listModelsAfter).toEqual(ROWS.keyedListing); + }); - it.skipIf(!PINNED)( - "resolveModelInfo, resolveCallConfig and the wire match the in-process rows for the decoration state the probe recorded (pinned pi)", - () => { - // The loader starts rows in parallel (F17), so whether the plugin's - // decoration wrapped pi-ai's plane adapter is a race: measured on the - // published binary it did not (the plugin applied about 70 ms first), - // which is no margin to pin. So the rows follow the recorded state, and - // `todayRowsFor` throws on a missing or unknown value. - console.info( - `[e2e-host] plane adapter decorated: ${String(out.planeAdapterDecorated)} (inferred order ${out.inferredOrder})` - ); - for (const [key, row] of Object.entries( - todayRowsFor(out.inferredOrder) - )) { - expect(out.modelInfo[key], key).toEqual(row.info); - expect(out.callConfig[key], key).toBe(row.callConfig); - expect(out.dispatch[key], key).toEqual({ - finish: row.finish, - sent: row.sent, - }); - } + it("resolveModelInfo, resolveCallConfig and the wire match the in-process rows of this cell, for the decoration state the probe recorded", (context) => { + if (!rowsHold) { + context.skip(); } - ); + // The loader starts rows in parallel (F17), so whether the plugin's + // decoration wrapped pi-ai's plane adapter is a race: measured on the + // published binary it did not (the plugin applied about 70 ms first), + // which is no margin to pin. So the rows follow the recorded state, and + // `todayRowsFor` throws on a missing or unknown value. + console.info( + `[e2e-host] plane adapter decorated: ${String(out.planeAdapterDecorated)} (inferred order ${out.inferredOrder})` + ); + for (const [key, row] of Object.entries( + todayRowsFor(out.inferredOrder, ROWS_CELL) + )) { + expect(out.modelInfo[key], key).toEqual(row.info); + expect(out.callConfig[key], key).toBe(row.callConfig); + expect(out.dispatch[key], key).toEqual({ + finish: row.finish, + sent: row.sent, + }); + } + }); }); it("[BASELINE] the settings page reads every plane the profile declares", () => { @@ -477,24 +538,24 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { it("[KNOWN_DEFECT] a stored gap id blocks saving any new id on its plane", () => { // pi-ai's strict check names the FIRST stored id the installed pi cannot - // describe. The pinned pi lacks gpt-5.3-codex-spark, so it is named; a - // floating pi may have learned it, and then names the next undescribed id - // — so the floating cell asserts only that one of them blocks the save. + // describe. Every recorded cell's pi lacks gpt-5.3-codex-spark, so it is + // named; a newer pi may have learned it, and then names the next + // undescribed id — so that run asserts only that one of them blocks it. expect(out.saveBlocked.rejected).toMatch( - PINNED ? S0_TODAY.saveBlockedByStoredGap : SAVE_BLOCKED_BY_ANY_UNDESCRIBED + rowsHold ? ROWS.saveBlockedByStoredGap : SAVE_BLOCKED_BY_ANY_UNDESCRIBED ); }); it("[KNOWN_DEFECT] a catalog-unknown id cannot be saved, once the plane stores no gap id", () => { // Clearing only ZEN_UNDESCRIBED assumes the installed pi still describes - // the other opencode ids the profile declares. The floating cell asserts - // it too: a pi that retires one of them would fail here, and that is a - // retirement worth a red nightly rather than a row to skip. + // the other opencode ids the profile declares. A pi newer than the record + // asserts it too: one that retires one of them would fail here, and that + // is a retirement worth a red run rather than a row to skip. expect(out.saveCleared).toEqual({ accepted: true, dropped: [...ZEN_UNDESCRIBED], }); - expect(out.save.rejected).toMatch(S0_TODAY.saveRejectedUnknown); + expect(out.save.rejected).toMatch(ROWS.saveRejectedUnknown); expect(out.afterSave).toBeUndefined(); }); }); From e1d775ccaab6ca148ffaa99d7a0ba7924cafd067 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 16:01:07 +0000 Subject: [PATCH 06/11] docs: describe the host cells and the per-channel binary e2e CONTRIBUTING, the repo map and the plan still described one pinned, patched pi and a floating cell. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- AGENTS.md | 2 +- CONTRIBUTING.md | 24 +++++++++++++----------- docs/improvement-plan.md | 14 +++++++------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a6574f8a..ce492ebf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,7 +43,7 @@ Host bundle (`lib/index.mjs`), a thin `apply` barrel over small modules: - `src/index.ts` · `src/identity.ts` · `src/lifecycle.ts` — the public barrel, the identity constants, and the `apply()` composition (one installer per side effect, in load-bearing order). The barrel is the host's contract and nothing more: `apply`, `Config`, `inject`, `name` and the config types. Everything else is internal, and `test/public-api.test.ts` pins the list, so widening it is a reviewed change. - `test/support/src.ts` — the one door tests use into `src`: each name re-exported from the module that DEFINES it, never through the barrel, so a module move edits one file. `test/architecture.test.ts` enforces both halves — every name is declared in its target, and no test outside the named exemptions (`vi.mock` targets, fresh re-imports after `vi.resetModules()`, the architecture and public-API tests, the jsdom client tests, the `reset-process-state.ts` setup file) imports a `src` path, and no jsdom test loads the door. - `test/support/host-versions.ts` — the hosts the plugin supports, named once: one record per harness channel (`rc` = `dsh@latest`, `alpha` = `dsh@alpha`), each with its `@deepseek-ai` cluster, cordis, its loader, the bridge's pi RANGE, the newest pi that range resolves (the channel lockfile's) and the range's floor (the harness's own pin, which only the desktop app ships), plus which one the install under test is (`DSH_CHANNEL`, else the marker `scripts/use-channel.ts` leaves, else `alpha`). `test/host-versions.test.ts` checks the installed tree against the SELECTED record; `test/harness-channel.test.ts` checks the declarations and holds each committed lockfile (`pnpm-lock.yaml` for alpha, `channels/rc/pnpm-lock.yaml` for rc) to its record, including that `@deepseek-ai/schemastery` resolves to one copy inside the cluster's range. -- `test/support/host.ts` — the real host, in-process, in two tiers: **A** (`mountTierA`: a bare cordis `Context`, `LlmRuntime` and `dsh-llm-pi-ai` mounted with `{ providers }`) for the `internal/config` waterfall and pi-ai's credential paths, and **B** (`bootTierB`: dsh-app-boot's `boot()` with a temporary HOME and a test bundle) for the config editor, `settings.describe`, the volatile commit and the characterization of the plugin as it ships. Its fakes (`fakeCredentials`, `fakeLaunchEnvironment`, `recordingFetch`) keep every case offline and its credentials out of the environment (a boot blanks the two keys the plugin reads directly, should the shell export them, and restores them on dispose); `S0_PROFILE` is the contract fixture, `CHARACTERIZED_PROFILE` (S0 plus a declared catalog-unknown id) the characterization's, and `S0_TODAY` the measured per-model rows, tagged per facet, that the binary e2e shares (`todayRowsFor` picks them by the apply order the probe infers from whether the plugin's decoration wrapped pi-ai's plane adapter). Tier A has no loader, so this plugin mounted there cannot find pi-ai and mounts no hidden route and no Google bridge (whose adapter class comes from the same loader entry, `loadPiAiAdapterHost`); `test/host-harness.test.ts` exercises the builder options no contract or characterization case passes, and tier B's two failure paths (an invalid first row, an exported key). Tier B's rows carry PACKAGE names through a stub module loader, never `cordis:` builtins, because the plugin finds pi-ai's entry by `options.name` (`piAiEntry` in `src/responses-provider.ts`) and a builtin-named row would drop the hidden-route mount from the characterization. It also fixes which of pi-ai and the plugin applies first (`order`), because the loader starts rows in parallel and the published binary applies the plugin first; the held row is released when the first one applies or its fiber settles, so an invalid first row fails instead of hanging the boot. +- `test/support/host.ts` — the real host, in-process, in two tiers: **A** (`mountTierA`: a bare cordis `Context`, `LlmRuntime` and `dsh-llm-pi-ai` mounted with `{ providers }`) for the `internal/config` waterfall and pi-ai's credential paths, and **B** (`bootTierB`: dsh-app-boot's `boot()` with a temporary HOME and a test bundle) for the config editor, `settings.describe`, the volatile commit and the characterization of the plugin as it ships. Its fakes (`fakeCredentials`, `fakeLaunchEnvironment`, `recordingFetch`) keep every case offline and its credentials out of the environment (a boot blanks the two keys the plugin reads directly, should the shell export them, and restores them on dispose); `S0_PROFILE` is the contract fixture, `CHARACTERIZED_PROFILE` (S0 plus a declared catalog-unknown id) the characterization's, and `S0_TODAY_BY_CELL` the measured per-model rows, tagged per facet, one table per host cell (`newest`, either channel's default install, and `rc-floor`/`alpha-floor`, the pis the desktop app ships; `S0_TODAY` is the installed cell's, and `forHost`/`forChannel` pick any other per-cell expectation), that the binary e2e shares (`todayRowsFor` picks them by the cell and by the apply order the probe infers from whether the plugin's decoration wrapped pi-ai's plane adapter). Tier A has no loader, so this plugin mounted there cannot find pi-ai and mounts no hidden route and no Google bridge (whose adapter class comes from the same loader entry, `loadPiAiAdapterHost`); `test/host-harness.test.ts` exercises the builder options no contract or characterization case passes, and tier B's two failure paths (an invalid first row, an exported key). Tier B's rows carry PACKAGE names through a stub module loader, never `cordis:` builtins, because the plugin finds pi-ai's entry by `options.name` (`piAiEntry` in `src/responses-provider.ts`) and a builtin-named row would drop the hidden-route mount from the characterization. It also fixes which of pi-ai and the plugin applies first (`order`), because the loader starts rows in parallel and the published binary applies the plugin first; the held row is released when the first one applies or its fiber settles, so an invalid first row fails instead of hanging the boot. - `test/support/keyless-matrix.ts` — the keyless 401-vs-404 probe matrix's cell table and classifier. Pure, though it lives in `test/support/` rather than `src/utils/`: a function of its arguments, with no code path that can set a credential header; `test/e2e/keyless-matrix.e2e.ts` sends the cells. - `test/e2e/host/` — the measurement bundle (`probe-plugin.mjs`, its manifest and patch) the `e2e-host` job links into a throwaway profile under a temporary HOME, through a relative `link:` path. `test/e2e/host-binary.e2e.ts` is its reader, gated on `DSH_HOST_E2E=1`; it installs no host itself and needs `DSH_HOST_PREFIX`, a prefix holding the installed one. - `scripts/report-host-drift.ts` — the nightly harness-drift report (CI's `host-drift` job, schedule and dispatch only): reads the registry the way npm resolves each channel and prints one `::warning::` per record field that moved, exiting 0 whatever it finds, because a channel moving is the harness's release, not a regression. Its rules are `scripts/host-drift.ts` (`hostDrift`, pure), held by `test/host-drift.test.ts` with known answers, a loopback registry and the job's evaluated condition. P-T3 turns the report into a proposed bump pull request. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37ea443f..bd65a5f6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ The e2e files, one job each: - `opencode-live.e2e.ts` — the only place the plugin meets the real API. It catches what a stub cannot: **the vendor changing the payload**. The meter parses `/zen/go/v1/usage` and the picker consumes an enriched `/models` listing, so both shapes are asserted against the live service. The unkeyed cases assert reachability (never a 404) and that our header set does not change the gateway's verdict on a request — compared against the same call without it, rather than hardcoding a status the vendor may tighten. - `patched-fetch-headers.e2e.ts` — a local `node:http` listener, so the _outgoing_ header set crosses a real socket. The live gateway cannot echo a request back; everywhere else in the suite the injected headers are asserted against a captured `fetch`. -- `keyless-matrix.e2e.ts` — the keyless 401-vs-404 matrix: two planes × four protocols × the legacy (`/zen(/go)/v1`) and modern (`/inference(/go)/…`) endpoint families, one PAID model per cell from the live catalog — never a free one, which the gateway serves keyless with a real completion. It **sends no key and reads none**, so the `e2e` job runs it on every pull request, and it retries a cell once on a 429, a 5xx or a transport error before reading it. A 404 means the path is gone, a 401/403 that it exists, a 410 or a 400 "Model is unavailable" that the model is. It asserts that all sixteen cells have a model and were sent, that every answer is readable, and that no cell the plugin dispatches to today with the default endpoint (`auto`, an alias of `modern`; `TODAY_DISPATCH`: completions on the LEGACY family, Responses, Messages and Google on the modern one) is gone — nor the modern completions cells P-R1 moves completions to (`NEXT_DISPATCH`, a labelled forward guard). The other legacy verdicts are recorded for P-R2 — among them the cells an `endpoint: legacy` user dispatches to (Responses and Messages on both planes, and Zen's Google), which are recorded, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell `TODAY_DISPATCH` already asserts. The cell table, both sets and the classifier are a pure module (`test/support/keyless-matrix.ts`), unit-tested offline in `test/keyless-matrix.test.ts`, which also derives `TODAY_DISPATCH` from the URLs `S0_TODAY` measured. +- `keyless-matrix.e2e.ts` — the keyless 401-vs-404 matrix: two planes × four protocols × the legacy (`/zen(/go)/v1`) and modern (`/inference(/go)/…`) endpoint families, one PAID model per cell from the live catalog — never a free one, which the gateway serves keyless with a real completion. It **sends no key and reads none**, so the `e2e` job runs it on every pull request, and it retries a cell once on a 429, a 5xx or a transport error before reading it. A 404 means the path is gone, a 401/403 that it exists, a 410 or a 400 "Model is unavailable" that the model is. It asserts that all sixteen cells have a model and were sent, that every answer is readable, and that no cell the plugin dispatches to today with the default endpoint (`auto`, an alias of `modern`; `TODAY_DISPATCH`: completions on the LEGACY family, where the rc floor's pi 1.1.0 files them, Responses, Messages and Google on the modern one) is gone — nor the modern completions cells pi 1.5.0 already sends them to and P-R1 moves every host to (`NEXT_DISPATCH`). The other legacy verdicts are recorded for P-R2 — among them the cells an `endpoint: legacy` user dispatches to (Responses and Messages on both planes, and Zen's Google), which are recorded, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell `TODAY_DISPATCH` already asserts. The cell table, both sets and the classifier are a pure module (`test/support/keyless-matrix.ts`), unit-tested offline in `test/keyless-matrix.test.ts`, which also derives `TODAY_DISPATCH` and `NEXT_DISPATCH` from the URLs each host cell's rows measured (alpha's floor, pi 1.3.0, files completions under the root family `/inference(/go)/v1`, which the matrix gains with P-R2). - `protocol-routing.e2e.ts` — one turn per protocol against the live gateway: the vendor-side claim that a model's `provider_npm` names its wire format, which a unit test can only read back from our own table. - `refusal-classification.e2e.ts` — the host's error classifier fed the gateway's measured refusals over a real `node:http` socket, chunked bodies included, after asserting the installed bundle still carries the branches the verdicts rely on. - `dispatch-levels.e2e.ts` — no network: the installed pi's own level rule against both decorators across the shipped catalog, so every reasoning level the picker offers is one dispatch accepts. @@ -52,7 +52,9 @@ The e2e files, one job each: ## The real-host harness -`test/support/host.ts` builds the host from the packages this repository installs, at the versions `test/support/host-versions.ts` names (pi 1.1.0 with the harness patch, the `@deepseek-ai` cluster at the published host's tag). The routing work is judged against it: the contract tests pin the host facts the design rests on that the host can answer in-process (`docs/improvement-plan.md`: F1, F2, F4, F5, F8, F10–F16, F18 and F20, each by number; F6 and F19's auth half (constructing a `PiAiAdapter`) are first exercised by P-R1; F19's export half is pinned in `host-shapes` (`recordKeyFor`); F17 is honoured, not pinned — tier B fixes the apply order and the binary e2e infers it; F9 is a fact about opencode's own client, outside this host), and the characterization rows record what today's plugin does. +`test/support/host.ts` builds the host from the packages this repository installs: the harness channel `test/support/host-versions.ts` selects (the `@deepseek-ai` cluster at that channel's published tag), with pi unpatched at the newest version the bridge's range admits, or at the range's floor after `scripts/use-channel.ts --floor`. The routing work is judged against it: the contract tests pin the host facts the design rests on that the host can answer in-process (`docs/improvement-plan.md`: F1, F2, F4, F5, F8, F10–F16, F18 and F20, each by number; F6 and F19's auth half (constructing a `PiAiAdapter`) are first exercised by P-R1; F19's export half is pinned in `host-shapes` (`recordKeyFor`); F17 is honoured, not pinned — tier B fixes the apply order and the binary e2e infers it; F9 is a fact about opencode's own client, outside this host), and the characterization rows record what today's plugin does. + +**Host cells.** The rows are measured on three cells (`HostCell` in `host.ts`): `newest`, either channel's default install (pi 1.5.0 today, one pi for both channels, so one column; a fixture case fails the day the two records disagree), and `rc-floor` (pi 1.1.0) and `alpha-floor` (pi 1.3.0), the pis the desktop app ships. Where the cells differ, a row carries one expectation per cell (`forHost`, keyed by the cell's role, never by a pi version) or per channel (`forChannel`, for what the `@deepseek-ai` cluster decides, such as pi-ai's own defaults), and every tier file runs on whichever cell is installed. CI runs the unit suite, tiers A and B included, on both channels' `newest` install on every pull request and on both floors nightly and on dispatch; locally, `scripts/use-channel.ts` switches the install and the same run reads the cell from the marker it leaves. The cells differ today in pi's catalog bases (the legacy family on 1.1.0, the root `/inference(/go)/v1` on 1.3.0, per-protocol roots on 1.5.0, which also dropped five Zen models), in which S0 ids pi can place (`PI_GAPS_BY_CELL`: 1.5.0 places both Go gap ids), in whether pi's Go provider carries Google (F10: not on 1.1.0), and in pi-ai's own defaults (the alpha cluster raised them to 1_000_000 / 128_000). Two tiers, both in-process: @@ -67,7 +69,7 @@ What every case does: - **Give tier-B cases 30 s**, and wait with a deadline (`waitForRoutes`, 10 s by default, so a route that never registers fails by name inside the hook's budget), never a fixed sleep: the plugin mounts its hidden routes asynchronously. - **A first row that fails does not hang the boot.** Tier B holds the second of pi-ai and the plugin until the first applies OR its fiber settles in any state, so a row whose config the schema rejects boots with that fiber FAILED (`fiber.await()` rethrows the real error) and the other row applied. -**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it) or `[INTENDED_CHANGE]` (right today; a phase changes it on purpose). A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY` in `host.ts`, which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon-contributor`'s call config) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. +**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one cell's pi lacks that the vendor's service needs: no phase of this plugin fixes it, the title names the channel and pi, and it is reported upstream, D26). A facet's tag is per cell: one that pi 1.5.0 already serves the way P-R1 keeps (completions on the per-protocol family, Go `gemini-4-argon` validating, Go `minimax-m3.1-flash` placed) is `[BASELINE]` on `newest` and keeps its floor tag on the floors. A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon`'s call config, on the floors only: pi 1.5.0 places the id in either order) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. **A catalog refresh can touch the rows.** `S0_TODAY`'s context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case compares each row's context window and effort ids to the bundled catalog's derivation and names the model that moved; when it fails on a refresh pull request, update the row in `S0_TODAY`. A phase that moves a non-`BASELINE` `info` facet off the catalog on purpose (the user's context window, a toggle model's Off/On) adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. @@ -75,24 +77,24 @@ What every case does: ### The `e2e-host` job -`.github/workflows/ci.yml`'s `e2e-host` job installs the published `@deepseek-ai/dsh` (at `HOST_VERSIONS.dsh`) into a throwaway prefix, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the temporary home's credentials document (`$DSH_HOME/.credentials.yaml`, shared by every profile under that home), boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. +`.github/workflows/ci.yml`'s `e2e-host` job installs each channel's published `@deepseek-ai/dsh` (its record's `dsh`) into a throwaway prefix with npm, the way a user installs it, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the temporary home's credentials document (`$DSH_HOME/.credentials.yaml`, shared by every profile under that home), boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. -- **Cells.** `pinned` forces pi, cordis and the loader to `HOST_VERSIONS` with npm `overrides` (the published host declares cordis and the loader as tilde ranges), applies the harness patch to pi with `patch -p1`, asserts all three installed versions (a cached prefix included), then asserts every row. `floating` takes the pi npm resolves for the published host. It skips the rows that depend on which ids pi describes (the keyed listing and the per-model rows) and still asserts the save rows. Those assume the floating pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage a red floating run as a pi change first. Its stored-gap save row asserts only that SOME undescribed id blocks the save, not which one; its probe JSON is the record. -- **Triggers.** Dispatch and the nightly run take both cells. A pull request takes `pinned`, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` or `patches/` (what the job installs and patches), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. +- **Cells, per channel, pi unpatched in both.** `npm` takes the pi npm resolves for the published host, what an npm user runs, and asserts the `newest` rows while that pi is the channel record's; a pi published since the record still has to sit in the bridge's range, but the rows that depend on which ids pi describes (the keyed listing and the per-model rows) are skipped with a warning, and the nightly `host-drift` job names the new pi. The save rows always run; they assume pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage that as a pi change first. `floor` forces pi to the range's floor with an npm `override` (the desktop app's pi), asserts it resolved, and asserts every row of the channel's floor cell. cordis and the loader are reported, not forced: the published host declares them as tilde ranges, and a new patch there is the channel's release. The installed `dsh` must be the channel record's, so a prefix of the wrong channel fails by name. +- **Triggers.** Dispatch and the nightly run take both cells of both channels. A pull request takes the `npm` cell of both channels, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` (what the job installs), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. -To run it locally, install the host into a prefix outside the checkout, then point the suite at it (placeholder paths; `pnpm` must be on `PATH`). The first line reads the versions from `test/support/host-versions.ts`, the same way the CI job does, so the snippet never names one itself: +To run it locally, install a channel's host into a prefix outside the checkout, then point the suite at it (placeholder paths; `pnpm` must be on `PATH`). The first line reads the channel's record from `test/support/host-versions.ts`, the same way the CI job does, so the snippet never names a version itself: ```sh -eval "$(node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS as v } from './test/support/host-versions.ts'; console.log('DSH=' + v.dsh + ' PI=' + v.pi + ' CORDIS=' + v.cordis + ' LOADER=' + v.loader)")" +export DSH_CHANNEL=alpha # or rc +eval "$(node --experimental-strip-types --input-type=module -e "import { CHANNELS } from './test/support/host-versions.ts'; const r = CHANNELS[process.env.DSH_CHANNEL]; console.log('DSH=' + r.dsh + ' PI_FLOOR=' + r.piFloor)")" mkdir -p /home/you/dsh-host -printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s", "@deepseek-ai/cordis": "%s", "@deepseek-ai/cordis-plugin-loader": "%s" } }\n' "$PI" "$CORDIS" "$LOADER" > /home/you/dsh-host/package.json +printf '{ "private": true }\n' > /home/you/dsh-host/package.json npm install --prefix /home/you/dsh-host "@deepseek-ai/dsh@$DSH" -patch -p1 -d /home/you/dsh-host/node_modules/@earendil-works/pi-ai < "patches/@earendil-works__pi-ai@$PI.patch" pnpm run build DSH_HOST_E2E=1 DSH_HOST_PREFIX=/home/you/dsh-host pnpm run test:e2e:host --run ``` -`DSH_HOST_PI_CELL=floating` runs the floating cell against an install without the override and the patch. +For the `floor` cell, use a fresh prefix whose `package.json` also carries `"overrides"` pinning `@earendil-works/pi-ai` to `$PI_FLOOR` before the install, and run the suite with `DSH_HOST_PI_CELL=floor`. ## Live profile loop diff --git a/docs/improvement-plan.md b/docs/improvement-plan.md index 5ff0b06d..96085997 100644 --- a/docs/improvement-plan.md +++ b/docs/improvement-plan.md @@ -25,7 +25,7 @@ Phases land as **stacked pull requests**. Each one is based on the previous phas | P1b | #107 | `chore:` hygiene | in review | **merged P1a**; takes `main`'s tool-fallback plane rule (D25) and keeps only the dead Anthropic branch's deletion; its catalogue fixture and `main`'s `test/fixtures/catalogs.ts` become one module behind the one door | | P2 | #108 | `feat:` a smaller public API | in review | **merged P1b**; re-retires the wrapper `main` revived in #135 (D9): `scripts/wrapper-forwarder.ts` and its test go again; the barrel stays the host's contract, and every name `main`'s tests reach (`parseAccountRef`, `callUsageRead`, `GO_MODEL_LIMITS`, …) comes through the one door | | P-T1a | #109 | `build:` pin the host | in review | **merged P2**; redone for both harness channels with no pi patch (D14'', D22''): one record per channel in `test/support/host-versions.ts`, the rc channel's committed lockfile and `scripts/use-channel.ts`, the unit and mock suites on both channels in CI, the nightly host-drift report, and the tripwire that keeps `src` off what the desktop's patched pi changes | -| P-T1b | #110 | `test:` a harness that is the real host | in review | pending: merges P-T1a; re-measured on both channels, newest and floor | +| P-T1b | #110 | `test:` a harness that is the real host | in review | **merged P-T1a**; re-measured on both channels, newest and floor: one row table per host cell (either channel's newest pi, and each floor), the binary e2e per channel (the `npm` cell on pull requests, the `floor` cell nightly), no pi patch anywhere | | P-H | #112 | `refactor:` one way to patch a host method | in review | pending: merges P-T1b | | P-R1 | #113 | `feat:` the plugin owns the plane routes | in review | pending: merges P-H; deletes the Gemini level wrapper and sets no default window or max tokens, pi-ai applies its own (D16, D17, D26) | | P-R2 | #114 | `fix:` the endpoint family, measured | in review | pending: merges P-R1; gains the L3-L5, L9, L10, L13 probes | @@ -367,7 +367,7 @@ What stays private is pinned by contract tests against the real host packages: - CI: the unit and mock suites run as a matrix over `[alpha, rc]` on every pull request, and on each channel's floor nightly and on dispatch. - CI: a `host-drift` job (schedule and dispatch, report-only, exit 0) runs `scripts/report-host-drift.ts`, whose pure half is `scripts/host-drift.ts`. - Dependabot ignores the cluster and pi (a bump follows D22''), and each lockfile is held to the manifest and workspace it was locked from, so a dependency change that skips `rc --relock` fails by name. -5. **P-T1b `test:` the real-host harness.** Additive only; in review. It landed `test/support/host.ts` (tier A: `Context` + `LlmRuntime` + pi-ai; tier B: app-boot `boot()` with a temporary HOME), the contract tests (`host-waterfall`, `host-shapes`, `host-credentials`, `host-editor-contracts`), the tagged characterization (`host-characterization`, with `S0_TODAY` shared by the binary e2e), the keyless matrix, the probe bundle and the `e2e-host` job. Two departures from the phase as first planned, both deliberate: tier B's rows carry package names through a stub module loader rather than `cordis:` builtins, because the plugin finds pi-ai by `options.name` (`piAiEntry` in `src/responses-provider.ts`); and tier B fixes the apply order (`plugin-first`, the binary's, or `pi-ai-first`), because the loader starts rows in parallel. Two narrower ones: the `e2e-host` PR scope is wider than first named (the job's own inputs, and until P-R1 the `src` modules that install the host patches and mounts whose behaviour depends on the real loader and apply order), and the floating cell skips the rows that depend on which ids pi describes. +5. **P-T1b `test:` the real-host harness.** Additive only; in review. It landed `test/support/host.ts` (tier A: `Context` + `LlmRuntime` + pi-ai; tier B: app-boot `boot()` with a temporary HOME), the contract tests (`host-waterfall`, `host-shapes`, `host-credentials`, `host-editor-contracts`), the tagged characterization (`host-characterization`, with `S0_TODAY` shared by the binary e2e), the keyless matrix, the probe bundle and the `e2e-host` job. Two departures from the phase as first planned, both deliberate: tier B's rows carry package names through a stub module loader rather than `cordis:` builtins, because the plugin finds pi-ai by `options.name` (`piAiEntry` in `src/responses-provider.ts`); and tier B fixes the apply order (`plugin-first`, the binary's, or `pi-ai-first`), because the loader starts rows in parallel. Two narrower ones: the `e2e-host` PR scope is wider than first named (the job's own inputs, and until P-R1 the `src` modules that install the host patches and mounts whose behaviour depends on the real loader and apply order), and an `npm` cell whose pi moved past its channel record skips the rows that depend on which ids pi describes. Restacked onto both channels (D14''), every tier file runs on whichever host cell is installed, and the rows carry one expectation per cell where the hosts differ (`S0_TODAY_BY_CELL`, `forHost`, `forChannel`): pi's catalog bases (legacy on 1.1.0, root on 1.3.0, per-protocol on 1.5.0), the S0 ids pi can place (1.5.0 places both Go gap ids, so their facets are `BASELINE` there), F10 (no Go Google on 1.1.0 only; a fact about pi the plugin's routing covers, so not a `HOST_LIMITATION`), and pi-ai's own defaults per cluster. No row here is a `HOST_LIMITATION`; P-R1 adds the first (D26). The `hasProviderApiKey` baseline (`api/session-controller/src/catalog.ts:97-116`) is a **restatement**: no installed copy reaches the function (the published `@deepseek-ai/dsh-api-session-controller` entry exports `SessionController`, `buildModelCatalog` and a few classes but not it, its tarball ships `lib/` and no `src/`, and the package is not a dependency here in any case), so `test/support/host.ts` restates it (`providerWithApiKey`) from the real parts tier B holds — `settings.describe({ redactSecrets: true })`, `llm.listConfigurableProviders()` and `credentials.describe` — keeping upstream's `credentialRef` grammar check as a `TypeError` (a host-editor-contracts case pins it), answering the provider id where upstream answers `true`, and omitting the `RemoteError` for a missing service tier B always has; both `host-editor-contracts` and `host-characterization` use it. The cases in `host-editor-contracts` boot WITHOUT the plugin, so they are host contracts that nothing in `src/` can move: with only the plane keys stored it answers true (the Zen plane); under a test-local strip listener it answers false, because the planes' `describe().value` is gone (F20), which upstream makes `initializeDefaultModel` replace the user's default model. That false is the host fact the D18 wrapper exists to counter, not a row any phase flips. The guard on the plugin is the `[BASELINE]` row in `host-characterization`: today's plugin as it ships, with only the plane keys stored, in both apply orders, answers the Zen plane, and pi-ai's `settings.describe` value lists both planes. P-R1 must keep that row green while its own strip is active. @@ -378,7 +378,7 @@ What stays private is pinned by contract tests against the real host packages: **Corrections the harness measured** (P-R1 must design for these): - **F8/F12, the save check.** An outermost listener that strips the planes and returns a copy does NOT shield pi-ai's own listener: pi-ai validates the value `next()` returns, which is the raw value the inner chain saw (`llm-pi-ai/src/index.ts:177-186`). Against its stripped previous config every plane entry reads as changed, so the strict check runs on the planes: an editor save is refused while a plane stores any gap id. The claim listener must either keep pi-ai's listener from seeing the planes (run the nested waterfall on the copy, with a re-entrancy guard) or veto the chain for pi-ai's fiber and validate in its place. - **F16, `!!js` in a provider.** pi-ai's listener runs `Config(raw)` on that same un-interpolated value, so a `!!js` node inside `providers` boots (evaluated) but makes every later save of the row fail with `expected string but got [object Object]`. The F16 pin is therefore taken on the plugin's own row. - - **`update()` runs the waterfall twice**, both as commits (`raw === _config`): once while ACTIVE with pi-ai's listener present (`fiber.ts:747`), once on the restart without it (`fiber.ts:655`). A first load and a `restart()` run it once, without pi-ai's listener. + - **`update()` runs the waterfall twice**, both as commits (`raw === _config`): once while ACTIVE with pi-ai's listener present (`fiber.ts:747` on rc, `776` on alpha), once on the restart without it (`fiber.ts:655` on rc, `682` on alpha). A first load and a `restart()` run it once, without pi-ai's listener. - **F15 holds** for a plane-only edit under the strip: no `loader/volatile-update`, no `llm/adapters-updated`, no restart. A sibling's edit emits one volatile update and two registry events (route replace and directory replace, `llm/src/index.ts:475,536`). - **The apply order changes today's rows.** On the published binary the plugin applies about 70 ms before pi-ai registers, so `decoratePlaneAdapters` (which wraps only what is registered when its effect runs) never wraps pi-ai's plane adapter. In that order Go `gemini-4-argon-contributor` answers `INVALID_CONFIG` — the plan's `KNOWN_DEFECT` holds — while the turn is still served on the Go Google base through the bridge. With pi-ai first it validates, and the Go plane's repaired base is Zen's (both planes share one adapter instance, wrapped twice). - **Zen `gpt-5.3-codex-spark`** is refused by `resolveCallConfig` (`INVALID_CONFIG`) yet dispatched to `/inference/openai/v1/responses` through the hidden route, and the user's `name`/`contextWindow` never reach `resolveModelInfo` (the catalog's 128000 does). @@ -637,15 +637,15 @@ Two community plugins solve the same problem on the same host: [`FishBottle7/ope **What it does:** -1. Installs `@deepseek-ai/dsh@$DSH_VERSION` into a throwaway prefix, cached by host version, with npm `overrides` forcing pi only in the floor cell. -2. Asserts the pi version that resolved. +1. Installs each channel's `@deepseek-ai/dsh` (its record's `dsh`) into a throwaway prefix with npm, pi unpatched: the npm cell is never cached (re-resolving is its point); the floor cell forces pi to the range's floor with an npm `override` and is cached by channel, `dsh` and that pi. +2. Asserts the installed `dsh` is the channel's, one pi copy, and the pi that resolved: the floor exactly; in the npm cell, inside the bridge's range, and the record's newest for the pi-dependent rows to run (a newer pi skips them with a warning). 3. Builds, and boots a temporary HOME whose profile links the checkout through a relative `link:`. 4. Runs `test/e2e/host/probe-plugin.mjs` together with `test/e2e/host-binary.e2e.ts`, which runs only when `DSH_HOST_E2E=1`. **Cells:** -- (a) each channel's published binary, with pi as npm resolves it (D14''), on PRs that touch the host seam; -- (b) each channel's floor pi, nightly and on dispatch. +- (a) each channel's published binary, with pi as npm resolves it (D14''), on PRs that touch the host seam, nightly and on dispatch; it asserts the `newest` cell's rows; +- (b) each channel's floor pi, nightly and on dispatch; it asserts that channel's floor cell's rows. ### The live gateway job (`OPENCODE_API_KEY` / `OPENCODE_GO_API_KEY`) From 4871d8703c8e225fe49a3811b1e5e5e1eb0eb9fa Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 23:20:41 +0000 Subject: [PATCH 07/11] test: name the host cell each row and dispatch set belongs to Review outcomes of the two-channel restack, without a channel bump: TODAY_DISPATCH is the rc floor's (the desktop app's pi 1.1.0) and NEXT_DISPATCH is where both channels' npm install already sends completions; the decoration case asserts per cell and route whether the wire went to the repaired base instead of an always-true comparison; the characterization header, the plan and CONTRIBUTING name the per-cell tables and the HOST_LIMITATION tag, which no P-T1b row carries; the probe artifact keeps a seven-day retention. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- .github/workflows/ci.yml | 1 + CONTRIBUTING.md | 2 +- docs/improvement-plan.md | 21 +++++++++++---------- test/e2e/host-binary.e2e.ts | 5 +++-- test/e2e/keyless-matrix.e2e.ts | 18 ++++++++++-------- test/host-characterization.test.ts | 27 ++++++++++++++++++--------- test/keyless-matrix.test.ts | 4 ++-- test/support/keyless-matrix.ts | 16 ++++++++++------ 8 files changed, 56 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cecd16cc..f20e8e5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -554,3 +554,4 @@ jobs: name: host-probe-${{ matrix.channel }}-${{ matrix.cell }} path: ${{ runner.temp }}/probe-${{ matrix.channel }}-${{ matrix.cell }}.json if-no-files-found: ignore + retention-days: 7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd65a5f6..42cb0a28 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ What every case does: **Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one cell's pi lacks that the vendor's service needs: no phase of this plugin fixes it, the title names the channel and pi, and it is reported upstream, D26). A facet's tag is per cell: one that pi 1.5.0 already serves the way P-R1 keeps (completions on the per-protocol family, Go `gemini-4-argon` validating, Go `minimax-m3.1-flash` placed) is `[BASELINE]` on `newest` and keeps its floor tag on the floors. A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon`'s call config, on the floors only: pi 1.5.0 places the id in either order) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. -**A catalog refresh can touch the rows.** `S0_TODAY`'s context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case compares each row's context window and effort ids to the bundled catalog's derivation and names the model that moved; when it fails on a refresh pull request, update the row in `S0_TODAY`. A phase that moves a non-`BASELINE` `info` facet off the catalog on purpose (the user's context window, a toggle model's Off/On) adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. +**A catalog refresh can touch the rows.** `S0_TODAY`'s context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case compares each row's context window and effort ids to the bundled catalog's derivation and names the model that moved; when it fails on a refresh pull request, update the row in `S0_RC_FLOOR` (the context windows and efforts every cell inherits), and in a cell's `withMoved` entry where that cell restates the row. A phase that moves a non-`BASELINE` `info` facet off the catalog on purpose (the user's context window, a toggle model's Off/On) adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. **The apply order is a race on the real host.** Tier B fixes it; the published binary does not. The rows depend on whether the plugin's adapter decoration wrapped pi-ai's plane adapter, so the probe records that and the order it implies (`inferredOrder` — inferred, not observed: a pi-ai restart after the plugin applied would read as plugin-first), and the binary e2e picks its rows with `todayRowsFor(inferredOrder)` instead of pinning one. diff --git a/docs/improvement-plan.md b/docs/improvement-plan.md index 96085997..ae8fc599 100644 --- a/docs/improvement-plan.md +++ b/docs/improvement-plan.md @@ -72,7 +72,7 @@ Phases land as **stacked pull requests**. Each one is based on the previous phas | D23 | **CI's freshness checks gate a pull request only on its own inputs.** A check's inputs are its script and every local module it loads, plus `src/identity.ts` for the CLI version; the generated tables never are. Vendor drift warns, and the nightly run proposes the refresh. The strict input lists in `test/freshness-scope.test.ts` are the record: `cli:version`'s grew by `scripts/vendor-version.ts` when `main` gave the check a pure half (#144), and the copy scan's generated-table skip moved into that half. | | D24 | **The debug gate is `main`'s rule:** naming `debugFile` writes both halves of the log; `debug` governs console output only. `main` chose it (#133), and its README example (`debug: false` with a `debugFile`) relies on it. P1a adopts it, and turns its "writes nothing while debug is off" test into one that asserts the rule. | | D25 | **The tool fallback's plane is `main`'s rule:** the request URL decides the plane, with a plane-blind lookup as the fallback (#138, `coreToolDecision`). P1b drops its route-mapped variant with no fallback. | -| D26 | **Host gaps are the host's job** (owner, 2026-11-03). The plugin never compensates for, re-implements or re-derives what the host's pi does, and never sniffs a channel or a version. So it deletes the Gemini thinking-level wrapper (`src/host/google-levels.ts`, which re-implemented pi 1.3's rule for an older pi), restates none of pi-ai's defaults (D17), and takes no base URL or protocol from pi's own catalog entries: the endpoint family table, the vendor's endpoints, is the only base source, and the protocol is the catalog's SDK, then the route's `api`, then completions (D16). It keeps what only the plugin can know (the gateway headers, the free-tier tool gate, the catalog's SDK to protocol, the endpoint families, per-plane credentials, the vendor's retirements) and what follows from owning the routes (the describe graft, listing declared ids). Google on the Go plane is routing, not a gap-fill: the plugin's adapter builds every catalog protocol for both planes through pi's public `createProvider`. Where a host's pi lacks something the vendor needs (Gemini 4 thinking levels on pi 1.1.0), that is a characterization row tagged `HOST_LIMITATION` with the channel and pi it applies to; live and mock cases that would exercise it skip by feature detection (pi's own exported predicate, never a version compare), a "Host limitations" list in the docs records it, and it is reported upstream. `test/architecture.test.ts` fails if `src` reads streamed tool-call deltas, partial arguments or thought signatures, so the desktop's patched pi stays invisible (D14''). P-T1a records this row and adds that tripwire. P-T1b tags only the characterization rows that already differ between the channels (F10). P-R1, the phase that removes the compensation, owns the rest: it deletes the wrapper and the restated defaults and rewrites the protocol order, and because the deletion is what exposes the limitation, it also adds the `HOST_LIMITATION` row (Gemini 4 thinking levels, channel `rc`, its floor pi `1.1.0`), the feature-detected skips in the mock case that inherits `google-thinking.test.ts`'s intents and in the live Gemini-level case (keyed on pi's own exported predicate answering for a Gemini 4 id, never on a version), the "Host limitations" list (in both READMEs, paired, or one doc they both link to), and the upstream report, recorded in its pull request. | +| D26 | **Host gaps are the host's job** (owner, 2026-11-03). The plugin never compensates for, re-implements or re-derives what the host's pi does, and never sniffs a channel or a version. So it deletes the Gemini thinking-level wrapper (`src/host/google-levels.ts`, which re-implemented pi 1.3's rule for an older pi), restates none of pi-ai's defaults (D17), and takes no base URL or protocol from pi's own catalog entries: the endpoint family table, the vendor's endpoints, is the only base source, and the protocol is the catalog's SDK, then the route's `api`, then completions (D16). It keeps what only the plugin can know (the gateway headers, the free-tier tool gate, the catalog's SDK to protocol, the endpoint families, per-plane credentials, the vendor's retirements) and what follows from owning the routes (the describe graft, listing declared ids). Google on the Go plane is routing, not a gap-fill: the plugin's adapter builds every catalog protocol for both planes through pi's public `createProvider`. Where a host's pi lacks something the vendor needs (Gemini 4 thinking levels on pi 1.1.0), that is a characterization row tagged `HOST_LIMITATION` with the channel and pi it applies to; live and mock cases that would exercise it skip by feature detection (pi's own exported predicate, never a version compare), a "Host limitations" list in the docs records it, and it is reported upstream. `test/architecture.test.ts` fails if `src` reads streamed tool-call deltas, partial arguments or thought signatures, so the desktop's patched pi stays invisible (D14''). P-T1a records this row and adds that tripwire. P-T1b gives the characterization rows one expectation per host cell where the cells differ (F10: no Go Google only on `rc`'s floor, pi `1.1.0`) and tags none of them `HOST_LIMITATION`; P-R1 adds the first. P-R1, the phase that removes the compensation, owns the rest: it deletes the wrapper and the restated defaults and rewrites the protocol order, and because the deletion is what exposes the limitation, it also adds the `HOST_LIMITATION` row (Gemini 4 thinking levels, channel `rc`, its floor pi `1.1.0`), the feature-detected skips in the mock case that inherits `google-thinking.test.ts`'s intents and in the live Gemini-level case (keyed on pi's own exported predicate answering for a Gemini 4 id, never on a version), the "Host limitations" list (in both READMEs, paired, or one doc they both link to), and the upstream report, recorded in its pull request. | ## What the host supports (read from source) @@ -328,16 +328,17 @@ What stays private is pinned by contract tests against the real host packages: - **Pinning phases open with a green pin**, which the PR shows unchanged: P-T1a/b, P-H, P5, P7, P8, P9, P10a and P-T2. - **P-T3 and P12 are exempt.** -**Characterization rows are tagged when they are recorded**, so a later phase flips named rows. The authoritative per-facet table is `S0_TODAY` in `test/support/host.ts` (each model's info, call-config verdict and wire carry their own tag); a phase's PR body cites the rows it flips from there. The three tags: +**Characterization rows are tagged when they are recorded**, so a later phase flips named rows. The authoritative per-facet tables are `S0_TODAY_BY_CELL` in `test/support/host.ts`, one per host cell (`S0_TODAY` is the installed cell's); each model's info, call-config verdict and wire carry their own tag, and a phase's PR body cites the rows it flips from there, per cell. The four tags: - `BASELINE`: right today and must survive every phase. No phase may move a `BASELINE` facet, except that P-R2's measured family flip, if it happens, may move the endpoint-family part of a wire facet; it re-tags each one it moves and names it in its PR body. - `KNOWN_DEFECT`, wrong today and a phase fixes it, for example: - - Go Gemini answers `INVALID_CONFIG` — in the published host's apply order (`plugin-first`); with pi-ai applying first it validates, so the row is order-dependent (`todayRowsFor`). P-R1 flips it and, in the same diff, drops `todayRowsFor`'s pi-ai-first override and turns the `[KNOWN_DEFECT]` apply-order case into a `[BASELINE]` "no facet depends on the apply order" check; + - Go Gemini answers `INVALID_CONFIG` — on the floor cells (rc's pi `1.1.0`, alpha's `1.3.0`, the desktop app's pi), in the published host's apply order (`plugin-first`); with pi-ai applying first it validates, so the row is order-dependent there (`todayRowsFor`). On `newest` (both channels' npm install, pi `1.5.0`) pi places the id and it validates in either order. P-R1 flips it and, in the same diff, drops `todayRowsFor`'s pi-ai-first override and turns the `[KNOWN_DEFECT]` apply-order case into a `[BASELINE]` "no facet depends on the apply order" check; - an unknown id is rejected; - toggle models show Minimal/Xhigh/Max. - `INTENDED_CHANGE`, right today and a phase changes it on purpose, for example: - - completions models move from `/zen(/go)/v1` to `/inference(/go)/openai/v1`; - - `decoratePlaneAdapters`' baseUrl repair never reaches the wire — a row of the `pi-ai-first` block only: in `plugin-first` the decoration never runs, and that row is a `KNOWN_DEFECT`. + - completions models move from pi's own catalog base (legacy `/zen(/go)/v1` on `rc-floor`, the root `/inference(/go)/v1` on `alpha-floor`) to `/inference(/go)/openai/v1`, which is already `BASELINE` on `newest` (pi `1.5.0` files them there); + - `decoratePlaneAdapters`' baseUrl repair never reaches the wire — a row of the `pi-ai-first` block only: in `plugin-first` the decoration never runs, and that row is a `KNOWN_DEFECT`. On `newest` the Zen plane's catalog base is the repaired base, so that one pair cannot tell the two apart and the case asserts it on the other routes and cells only. +- `HOST_LIMITATION` (D26): a host's pi lacks something the vendor needs, tagged with the channel and pi it applies to. It is recorded, skipped by feature detection where a case would exercise it, and reported upstream, never fixed in the plugin. No P-T1b row carries it (F10 is a per-cell fact the plugin's routing covers); P-R1 adds the first. 1. **P1a `fix:` the purge row's refusal and the endpoint copy.** The rest of what shipped broken in 1.4.1 was fixed on `main` (#133, #135) before this phase landed. - `settings-purge` shows the `message` of a refusal whose `error` is the host's `RemoteError` object. @@ -367,7 +368,7 @@ What stays private is pinned by contract tests against the real host packages: - CI: the unit and mock suites run as a matrix over `[alpha, rc]` on every pull request, and on each channel's floor nightly and on dispatch. - CI: a `host-drift` job (schedule and dispatch, report-only, exit 0) runs `scripts/report-host-drift.ts`, whose pure half is `scripts/host-drift.ts`. - Dependabot ignores the cluster and pi (a bump follows D22''), and each lockfile is held to the manifest and workspace it was locked from, so a dependency change that skips `rc --relock` fails by name. -5. **P-T1b `test:` the real-host harness.** Additive only; in review. It landed `test/support/host.ts` (tier A: `Context` + `LlmRuntime` + pi-ai; tier B: app-boot `boot()` with a temporary HOME), the contract tests (`host-waterfall`, `host-shapes`, `host-credentials`, `host-editor-contracts`), the tagged characterization (`host-characterization`, with `S0_TODAY` shared by the binary e2e), the keyless matrix, the probe bundle and the `e2e-host` job. Two departures from the phase as first planned, both deliberate: tier B's rows carry package names through a stub module loader rather than `cordis:` builtins, because the plugin finds pi-ai by `options.name` (`piAiEntry` in `src/responses-provider.ts`); and tier B fixes the apply order (`plugin-first`, the binary's, or `pi-ai-first`), because the loader starts rows in parallel. Two narrower ones: the `e2e-host` PR scope is wider than first named (the job's own inputs, and until P-R1 the `src` modules that install the host patches and mounts whose behaviour depends on the real loader and apply order), and an `npm` cell whose pi moved past its channel record skips the rows that depend on which ids pi describes. Restacked onto both channels (D14''), every tier file runs on whichever host cell is installed, and the rows carry one expectation per cell where the hosts differ (`S0_TODAY_BY_CELL`, `forHost`, `forChannel`): pi's catalog bases (legacy on 1.1.0, root on 1.3.0, per-protocol on 1.5.0), the S0 ids pi can place (1.5.0 places both Go gap ids, so their facets are `BASELINE` there), F10 (no Go Google on 1.1.0 only; a fact about pi the plugin's routing covers, so not a `HOST_LIMITATION`), and pi-ai's own defaults per cluster. No row here is a `HOST_LIMITATION`; P-R1 adds the first (D26). +5. **P-T1b `test:` the real-host harness.** Additive only; in review. It landed `test/support/host.ts` (tier A: `Context` + `LlmRuntime` + pi-ai; tier B: app-boot `boot()` with a temporary HOME), the contract tests (`host-waterfall`, `host-shapes`, `host-credentials`, `host-editor-contracts`), the tagged characterization (`host-characterization`, with `S0_TODAY_BY_CELL` shared by the binary e2e, which reads the table of the cell it installed), the keyless matrix, the probe bundle and the `e2e-host` job. Two departures from the phase as first planned, both deliberate: tier B's rows carry package names through a stub module loader rather than `cordis:` builtins, because the plugin finds pi-ai by `options.name` (`piAiEntry` in `src/responses-provider.ts`); and tier B fixes the apply order (`plugin-first`, the binary's, or `pi-ai-first`), because the loader starts rows in parallel. Two narrower ones: the `e2e-host` PR scope is wider than first named (the job's own inputs, and until P-R1 the `src` modules that install the host patches and mounts whose behaviour depends on the real loader and apply order), and an `npm` cell whose pi moved past its channel record skips the rows that depend on which ids pi describes. Restacked onto both channels (D14''), every tier file runs on whichever host cell is installed, and the rows carry one expectation per cell where the hosts differ (`S0_TODAY_BY_CELL`, `forHost`, `forChannel`): pi's catalog bases (legacy on 1.1.0, root on 1.3.0, per-protocol on 1.5.0), the S0 ids pi can place (1.5.0 places both Go gap ids, so their facets are `BASELINE` there), F10 (no Go Google on 1.1.0 only; a fact about pi the plugin's routing covers, so not a `HOST_LIMITATION`), and pi-ai's own defaults per cluster. No row here is a `HOST_LIMITATION`; P-R1 adds the first (D26). The `hasProviderApiKey` baseline (`api/session-controller/src/catalog.ts:97-116`) is a **restatement**: no installed copy reaches the function (the published `@deepseek-ai/dsh-api-session-controller` entry exports `SessionController`, `buildModelCatalog` and a few classes but not it, its tarball ships `lib/` and no `src/`, and the package is not a dependency here in any case), so `test/support/host.ts` restates it (`providerWithApiKey`) from the real parts tier B holds — `settings.describe({ redactSecrets: true })`, `llm.listConfigurableProviders()` and `credentials.describe` — keeping upstream's `credentialRef` grammar check as a `TypeError` (a host-editor-contracts case pins it), answering the provider id where upstream answers `true`, and omitting the `RemoteError` for a missing service tier B always has; both `host-editor-contracts` and `host-characterization` use it. The cases in `host-editor-contracts` boot WITHOUT the plugin, so they are host contracts that nothing in `src/` can move: with only the plane keys stored it answers true (the Zen plane); under a test-local strip listener it answers false, because the planes' `describe().value` is gone (F20), which upstream makes `initializeDefaultModel` replace the user's default model. That false is the host fact the D18 wrapper exists to counter, not a row any phase flips. The guard on the plugin is the `[BASELINE]` row in `host-characterization`: today's plugin as it ships, with only the plane keys stored, in both apply orders, answers the Zen plane, and pi-ai's `settings.describe` value lists both planes. P-R1 must keep that row green while its own strip is active. @@ -380,11 +381,11 @@ What stays private is pinned by contract tests against the real host packages: - **F16, `!!js` in a provider.** pi-ai's listener runs `Config(raw)` on that same un-interpolated value, so a `!!js` node inside `providers` boots (evaluated) but makes every later save of the row fail with `expected string but got [object Object]`. The F16 pin is therefore taken on the plugin's own row. - **`update()` runs the waterfall twice**, both as commits (`raw === _config`): once while ACTIVE with pi-ai's listener present (`fiber.ts:747` on rc, `776` on alpha), once on the restart without it (`fiber.ts:655` on rc, `682` on alpha). A first load and a `restart()` run it once, without pi-ai's listener. - **F15 holds** for a plane-only edit under the strip: no `loader/volatile-update`, no `llm/adapters-updated`, no restart. A sibling's edit emits one volatile update and two registry events (route replace and directory replace, `llm/src/index.ts:475,536`). - - **The apply order changes today's rows.** On the published binary the plugin applies about 70 ms before pi-ai registers, so `decoratePlaneAdapters` (which wraps only what is registered when its effect runs) never wraps pi-ai's plane adapter. In that order Go `gemini-4-argon-contributor` answers `INVALID_CONFIG` — the plan's `KNOWN_DEFECT` holds — while the turn is still served on the Go Google base through the bridge. With pi-ai first it validates, and the Go plane's repaired base is Zen's (both planes share one adapter instance, wrapped twice). + - **The apply order changes today's rows.** On the published binary the plugin applies about 70 ms before pi-ai registers, so `decoratePlaneAdapters` (which wraps only what is registered when its effect runs) never wraps pi-ai's plane adapter. In that order, on the floor cells (rc's pi `1.1.0`, alpha's `1.3.0`, the desktop app's pi), Go `gemini-4-argon` answers `INVALID_CONFIG` — the plan's `KNOWN_DEFECT` holds — while the turn is still served on the Go Google base through the bridge; pi `1.5.0`, both channels' npm install, places the id in either order. With pi-ai first it validates, and the Go plane's repaired base is Zen's (both planes share one adapter instance, wrapped twice). - **Zen `gpt-5.3-codex-spark`** is refused by `resolveCallConfig` (`INVALID_CONFIG`) yet dispatched to `/inference/openai/v1/responses` through the hidden route, and the user's `name`/`contextWindow` never reach `resolveModelInfo` (the catalog's 128000 does). - **Listings.** Before a key is captured the listing patch hides every paid model on both planes (`[]`); after one, only pi-resolved ids are listed and the gap ids are not merged back. - **A stored gap id blocks every edit of its plane**, `displayName` included: the strict check names the stored id first, ahead of the one being saved. - - **Shapes.** A resolved profile also carries the profile's pass-through fields (`compat`, `defaultContextWindow`, `defaultInput`, `defaultMaxTokens`, `headers`, `modelOverrides`, `thinkingBudgets`), and a built pi provider exposes `getModels()` but no `api` map, so F10 is pinned by behaviour (pi's own "no API implementation" error) rather than by its keys. + - **Shapes.** A resolved profile also carries the profile's pass-through fields (`compat`, `defaultContextWindow`, `defaultInput`, `defaultMaxTokens`, `headers`, `modelOverrides`, `thinkingBudgets`), and a built pi provider exposes `getModels()` but no `api` map, so on the `rc-floor` cell (pi `1.1.0`) F10 is pinned by behaviour (pi's own "no API implementation" error) rather than by its keys; on pi `1.3.0` and `1.5.0` it is pinned by the API set `getModels()` answers, which includes Google. 6. **P-H `refactor:` one way to patch a host method.** `host/patch-method.ts` and `test/reload.test.ts` (apply A → apply B → dispose A → B intact → dispose B → originals). 7. **P-R1 `feat:` the plugin owns the plane routes.** @@ -633,7 +634,7 @@ Two community plugins solve the same problem on the same host: [`FishBottle7/ope ### The `e2e-host` job (`ci.yml`) -**When it runs:** on dispatch, nightly, and on PRs that touch `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each PR by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts`, `test/support/host-versions.ts`, `patches/**`, `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the pack config that builds `lib/index.mjs`, which only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step, since a workflow-level path filter would silence every job). +**When it runs:** on dispatch, nightly, and on PRs that touch `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each PR by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts`, `test/support/host-versions.ts`, `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the pack config that builds `lib/index.mjs`, which only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step, since a workflow-level path filter would silence every job). **What it does:** @@ -649,7 +650,7 @@ Two community plugins solve the same problem on the same host: [`FishBottle7/ope ### The live gateway job (`OPENCODE_API_KEY` / `OPENCODE_GO_API_KEY`) -- **Every PR:** the keyless matrix. A 404 means the path is gone; a 400 "Model is unavailable" or a 410 means the model is gone. It asserts that every cell has a model and was sent, and that no cell today's plugin dispatches to with the default endpoint (`auto`, an alias of `modern`: legacy completions; modern Responses, Messages and Google) nor P-R1's modern completions target answers 404; the other legacy verdicts are P-R2's record. `endpoint: legacy` users dispatch Responses and Messages on both planes, and Zen's Google, to the legacy cells, which are recorded for P-R2, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell already asserted. +- **Every PR:** the keyless matrix. A 404 means the path is gone; a 400 "Model is unavailable" or a 410 means the model is gone. It asserts that every cell has a model and was sent, and that no cell the plugin dispatches to with the default endpoint (`auto`, an alias of `modern`) answers 404, per host cell: on the rc floor, the desktop app's pi 1.1.0, legacy completions and modern Responses, Messages and Google (`TODAY_DISPATCH`); on both channels' npm install, pi 1.5.0, modern completions too (`NEXT_DISPATCH`), where P-R1 moves every host; the other legacy verdicts are P-R2's record. `endpoint: legacy` users dispatch Responses and Messages on both planes, and Zen's Google, to the legacy cells, which are recorded for P-R2, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell already asserted. - **PRs that touch `src/host/**`, and nightly:** minimal keyed probes, one per plane × protocol, with a small `max_tokens`. - **Nightly and on dispatch:** the full keyed suites. diff --git a/test/e2e/host-binary.e2e.ts b/test/e2e/host-binary.e2e.ts index 9625fcc4..55150ead 100644 --- a/test/e2e/host-binary.e2e.ts +++ b/test/e2e/host-binary.e2e.ts @@ -10,8 +10,9 @@ * The probe records what the host serves and exits the process; this file reads * its JSON. * - * The assertions are TODAY's behaviour, read from the same `S0_TODAY` table the - * in-process characterization pins, so the binary and the harness cannot drift + * The assertions are TODAY's behaviour, read from `S0_TODAY_BY_CELL` for the + * cell this run installed, the same tables the in-process characterization + * pins, so the binary and the harness cannot drift * apart unnoticed. P-R1 flips the rows its PR names. The loader starts rows in * parallel, so which of pi-ai and the plugin applies first is a race. The rows * depend on whether the plugin's adapter decoration wrapped pi-ai's plane diff --git a/test/e2e/keyless-matrix.e2e.ts b/test/e2e/keyless-matrix.e2e.ts index c803deef..9d831642 100644 --- a/test/e2e/keyless-matrix.e2e.ts +++ b/test/e2e/keyless-matrix.e2e.ts @@ -14,12 +14,14 @@ * - every one of the sixteen cells has a model and was SENT, and every answer * is one the matrix can read — an empty or reshaped catalog fails here * rather than measuring nothing; - * - no cell today's plugin dispatches to with the default endpoint (`auto`, - * an alias of `modern`; `TODAY_DISPATCH`: completions on the legacy family, - * the hidden routes' Responses, Messages and Google on the modern one) - * answers "path gone"; - * - nor do the modern completions cells (`NEXT_DISPATCH`), where P-R1 moves - * completions: a forward guard, labelled as such when it fails. + * - no cell the plugin dispatches to with the default endpoint (`auto`, an + * alias of `modern`) on the rc floor, the desktop app's pi 1.1.0, answers + * "path gone" (`TODAY_DISPATCH`: completions on the legacy family, the + * hidden routes' Responses, Messages and Google on the modern one); + * - nor do the modern completions cells (`NEXT_DISPATCH`), where both + * channels' npm install (pi 1.5.0) already sends completions and where + * P-R1 moves every host: a 404 there breaks every npm user's completions + * today, and the failure says so. * * The other legacy verdicts are recorded, not asserted: P-R2 decides the * family from them. They include the cells an `endpoint: legacy` user @@ -169,14 +171,14 @@ describe.skipIf(!LIVE || MIRRORED)( ); expect( goneToday, - `a path today's plugin dispatches to is gone: ${JSON.stringify(goneToday)}` + `a path the plugin dispatches to on the rc floor (the desktop app's pi 1.1.0) is gone: ${JSON.stringify(goneToday)}` ).toEqual([]); const goneNext = rows.filter( (row) => NEXT_DISPATCH.has(row.cell) && row.verdict === "path-gone" ); expect( goneNext, - `forward guard: a path P-R1 moves completions to is gone: ${JSON.stringify(goneNext)}` + `a path both channels' npm install (pi 1.5.0) dispatches completions to, and P-R1 moves every host to, is gone: ${JSON.stringify(goneNext)}` ).toEqual([]); }, 16 * 2 * (PROBE_TIMEOUT_MS + RETRY_AFTER_MS) + PROBE_TIMEOUT_MS diff --git a/test/host-characterization.test.ts b/test/host-characterization.test.ts index cf4b46e4..47e03ad7 100644 --- a/test/host-characterization.test.ts +++ b/test/host-characterization.test.ts @@ -12,15 +12,18 @@ * * - `[KNOWN_DEFECT]` is wrong today, and a phase fixes it; * - `[INTENDED_CHANGE]` is right today, and a phase changes it on purpose; - * - `[BASELINE]` must survive every phase. + * - `[BASELINE]` must survive every phase; + * - `[HOST_LIMITATION]` is something one cell's pi lacks that the vendor's + * service needs, named with its channel and pi, reported upstream and never + * fixed here (D26). No row in this file carries it yet: P-R1 adds the first. * * Where the hosts differ, a row carries one expectation per cell * (`S0_TODAY_BY_CELL`, `forHost`), and a title names the cell's tag: a facet * pi 1.5.0 already serves the way P-R1 keeps is `[BASELINE]` on `newest` and * still `[INTENDED_CHANGE]` or `[KNOWN_DEFECT]` on a floor. * - * The per-model expectations live in `S0_TODAY` (`support/host.ts`), which - * the binary e2e reads too, so the in-process host and the published binary + * The per-model expectations live in `S0_TODAY_BY_CELL` (`support/host.ts`; + * `S0_TODAY` is the installed cell's table), which the binary e2e reads too, so the in-process host and the published binary * cannot drift apart unnoticed. Each model's three facets (model info, the * call-config verdict, the wire) carry their own tag there, and this file * generates one test per model and facet titled with that tag, in each apply @@ -752,7 +755,7 @@ describe( await expectSignInReadsTheZenPlane(); }); - it("[INTENDED_CHANGE] the adapter decoration runs, and the base it repairs is not what dispatch uses: the wire carries pi's own catalog base", async () => { + it("[INTENDED_CHANGE] the adapter decoration runs, and the base it repairs is not what dispatch uses: the wire carries pi's own catalog base (on newest, Zen's catalog base is the repaired one, so only Go shows it there)", async () => { const adapter = planeAdapter(); expect(Object.hasOwn(adapter, "prepareCall")).toBe(true); for (const route of PLANES) { @@ -769,11 +772,17 @@ describe( }); const { drained, requests } = await sentBy(route, "deepseek-v4-pro"); expect(requests.map((request) => request.url)).toEqual([wireUrl]); - // Measured, not compared as literals: the URL the turn sent starts - // with the base the decorated adapter prepared only where pi's own - // catalog base happens to be the same one (Zen on newest). - expect(requests[0]?.url.startsWith(String(prepared.baseUrl))).toBe( - String(wireUrl).startsWith(decorated) + // The claim, wherever a URL can show it: the turn did not go to the + // repaired base. That holds on every route of both floors and on Go + // on newest. On newest pi 1.5.0's own Zen catalog base IS the + // repaired base, so that one pair sends there whichever base dispatch + // used and cannot tell them apart; the expectation says so. + expect(String(requests[0]?.url).startsWith(decorated)).toBe( + forHost({ + "alpha-floor": false, + newest: route === "opencode", + "rc-floor": false, + }) ); expect(drained.finish).toEqual({ code: "AUTH", kind: "error" }); } diff --git a/test/keyless-matrix.test.ts b/test/keyless-matrix.test.ts index d4b5c136..b242615a 100644 --- a/test/keyless-matrix.test.ts +++ b/test/keyless-matrix.test.ts @@ -324,8 +324,8 @@ describe("keyless matrix · the cells the live run asserts", () => { const hidden = [...TODAY_DISPATCH].filter( (cell) => !cell.includes("openai-completions") ); - // pi 1.5.0 (newest) files completions under the modern family: exactly - // the forward-guard cells P-R1 targets. + // pi 1.5.0 (newest, both channels' npm install) files completions under + // the modern family: exactly the cells P-R1 moves every host to. const newest = reachedOn("newest"); expect(newest.unmatched).toEqual([]); expect(newest.reached).toEqual([...hidden, ...NEXT_DISPATCH].toSorted()); diff --git a/test/support/keyless-matrix.ts b/test/support/keyless-matrix.ts index 591bd412..9212cf46 100644 --- a/test/support/keyless-matrix.ts +++ b/test/support/keyless-matrix.ts @@ -56,9 +56,10 @@ export const cellKey = ( ): string => `${cell.plane} ${cell.protocol} ${cell.family}`; /** - * The cells TODAY's plugin sends turns to with the default endpoint (`auto`, - * an alias of `modern`), on both planes, on the oldest supported host (the rc - * channel's floor, pi 1.1.0): completions on pi's LEGACY bases + * The cells the plugin sends turns to with the default endpoint (`auto`, an + * alias of `modern`), on both planes, on the oldest supported host: the rc + * channel's floor, pi 1.1.0, which the desktop app ships. Completions go to + * pi's LEGACY bases * (`/zen(/go)/v1/chat/completions`, pi-ai's own plane route) and the three * hidden routes' MODERN paths (Responses, Messages, Google). A path-gone * verdict on one of these breaks a shipped model in the default @@ -86,9 +87,12 @@ export const TODAY_DISPATCH: ReadonlySet = new Set( ); /** - * The cells P-R1 moves traffic to that nothing dispatches to yet: completions - * on the modern family (`/inference(/go)/openai/v1`). The live matrix asserts - * them too, as P-R1's forward guard, and labels them so in its failure. + * Completions on the modern family (`/inference(/go)/openai/v1`): where both + * channels' default install (the npm install, pi 1.5.0) already sends every + * completions turn, and where P-R1 moves every host, the floors included. A + * path-gone verdict here breaks completions for every npm user today, so the + * live matrix asserts these cells as it does {@link TODAY_DISPATCH}, and its + * failure says which hosts it breaks. */ export const NEXT_DISPATCH: ReadonlySet = new Set( (["opencode", "opencode-go"] as const).map((plane) => From 79ef58a76e3f8c5f9f728844c7f3119e1742238c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 23:33:45 +0000 Subject: [PATCH 08/11] test: record the alpha host's broken save as a host limitation The alpha channel's npm install of 0.2.2-alpha.2 now resolves cordis 4.0.6-alpha.2, whose new internal/update call shape breaks the host's own dsh-app-boot listener: every config-editor save throws before any plugin runs, so both alpha cells of the binary e2e failed their save rows. The plugin cannot fix that (D26). The probe now names the error class and the package that threw it, and the e2e records a HOST_LIMITATION row and skips the save rows when the host's own code throws, detected from the answer and never from a version, so a fixed host runs them again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- CONTRIBUTING.md | 3 +- docs/improvement-plan.md | 6 ++- test/e2e/host-binary.e2e.ts | 89 ++++++++++++++++++++++++++++++---- test/e2e/host/probe-plugin.mjs | 31 +++++++++++- 4 files changed, 115 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 42cb0a28..555bcb15 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ What every case does: - **Give tier-B cases 30 s**, and wait with a deadline (`waitForRoutes`, 10 s by default, so a route that never registers fails by name inside the hook's budget), never a fixed sleep: the plugin mounts its hidden routes asynchronously. - **A first row that fails does not hang the boot.** Tier B holds the second of pi-ai and the plugin until the first applies OR its fiber settles in any state, so a row whose config the schema rejects boots with that fiber FAILED (`fiber.await()` rethrows the real error) and the other row applied. -**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one cell's pi lacks that the vendor's service needs: no phase of this plugin fixes it, the title names the channel and pi, and it is reported upstream, D26). A facet's tag is per cell: one that pi 1.5.0 already serves the way P-R1 keeps (completions on the per-protocol family, Go `gemini-4-argon` validating, Go `minimax-m3.1-flash` placed) is `[BASELINE]` on `newest` and keeps its floor tag on the floors. A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon`'s call config, on the floors only: pi 1.5.0 places the id in either order) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. +**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one cell's host lacks or breaks, its pi or its own code, that no phase of this plugin fixes: the title names the channel and the host, and it is reported upstream, D26). A facet's tag is per cell: one that pi 1.5.0 already serves the way P-R1 keeps (completions on the per-protocol family, Go `gemini-4-argon` validating, Go `minimax-m3.1-flash` placed) is `[BASELINE]` on `newest` and keeps its floor tag on the floors. A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon`'s call config, on the floors only: pi 1.5.0 places the id in either order) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. **A catalog refresh can touch the rows.** `S0_TODAY`'s context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case compares each row's context window and effort ids to the bundled catalog's derivation and names the model that moved; when it fails on a refresh pull request, update the row in `S0_RC_FLOOR` (the context windows and efforts every cell inherits), and in a cell's `withMoved` entry where that cell restates the row. A phase that moves a non-`BASELINE` `info` facet off the catalog on purpose (the user's context window, a toggle model's Off/On) adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. @@ -80,6 +80,7 @@ What every case does: `.github/workflows/ci.yml`'s `e2e-host` job installs each channel's published `@deepseek-ai/dsh` (its record's `dsh`) into a throwaway prefix with npm, the way a user installs it, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the temporary home's credentials document (`$DSH_HOME/.credentials.yaml`, shared by every profile under that home), boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. - **Cells, per channel, pi unpatched in both.** `npm` takes the pi npm resolves for the published host, what an npm user runs, and asserts the `newest` rows while that pi is the channel record's; a pi published since the record still has to sit in the bridge's range, but the rows that depend on which ids pi describes (the keyed listing and the per-model rows) are skipped with a warning, and the nightly `host-drift` job names the new pi. The save rows always run; they assume pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage that as a pi change first. `floor` forces pi to the range's floor with an npm `override` (the desktop app's pi), asserts it resolved, and asserts every row of the channel's floor cell. cordis and the loader are reported, not forced: the published host declares them as tilde ranges, and a new patch there is the channel's release. The installed `dsh` must be the channel record's, so a prefix of the wrong channel fails by name. +- **cordis and the loader are not pinned, and the host's own failures are recorded, not worked around.** The published host declares both as `~` ranges, so each cell gets the newest patch npm resolves, as a user does, and the job reports the versions. Where that leaves the host unable to run its own code, the run records a `[HOST_LIMITATION]` row and skips the rows that need the broken path, detecting it from what the host answered (a `TypeError` thrown by a `@deepseek-ai/dsh-*` package, which the probe names) and never from a version (D26). Today that is alpha's `0.2.2-alpha.2` under cordis `4.0.6-alpha.2`: its `dsh-app-boot` registers an `internal/update` listener in the old call shape, so every config-editor save throws `next is not a function` before any plugin runs, and the two save rows skip on both alpha cells. - **Triggers.** Dispatch and the nightly run take both cells of both channels. A pull request takes the `npm` cell of both channels, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` (what the job installs), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. To run it locally, install a channel's host into a prefix outside the checkout, then point the suite at it (placeholder paths; `pnpm` must be on `PATH`). The first line reads the channel's record from `test/support/host-versions.ts`, the same way the CI job does, so the snippet never names a version itself: diff --git a/docs/improvement-plan.md b/docs/improvement-plan.md index ae8fc599..2aaf45c1 100644 --- a/docs/improvement-plan.md +++ b/docs/improvement-plan.md @@ -72,7 +72,7 @@ Phases land as **stacked pull requests**. Each one is based on the previous phas | D23 | **CI's freshness checks gate a pull request only on its own inputs.** A check's inputs are its script and every local module it loads, plus `src/identity.ts` for the CLI version; the generated tables never are. Vendor drift warns, and the nightly run proposes the refresh. The strict input lists in `test/freshness-scope.test.ts` are the record: `cli:version`'s grew by `scripts/vendor-version.ts` when `main` gave the check a pure half (#144), and the copy scan's generated-table skip moved into that half. | | D24 | **The debug gate is `main`'s rule:** naming `debugFile` writes both halves of the log; `debug` governs console output only. `main` chose it (#133), and its README example (`debug: false` with a `debugFile`) relies on it. P1a adopts it, and turns its "writes nothing while debug is off" test into one that asserts the rule. | | D25 | **The tool fallback's plane is `main`'s rule:** the request URL decides the plane, with a plane-blind lookup as the fallback (#138, `coreToolDecision`). P1b drops its route-mapped variant with no fallback. | -| D26 | **Host gaps are the host's job** (owner, 2026-11-03). The plugin never compensates for, re-implements or re-derives what the host's pi does, and never sniffs a channel or a version. So it deletes the Gemini thinking-level wrapper (`src/host/google-levels.ts`, which re-implemented pi 1.3's rule for an older pi), restates none of pi-ai's defaults (D17), and takes no base URL or protocol from pi's own catalog entries: the endpoint family table, the vendor's endpoints, is the only base source, and the protocol is the catalog's SDK, then the route's `api`, then completions (D16). It keeps what only the plugin can know (the gateway headers, the free-tier tool gate, the catalog's SDK to protocol, the endpoint families, per-plane credentials, the vendor's retirements) and what follows from owning the routes (the describe graft, listing declared ids). Google on the Go plane is routing, not a gap-fill: the plugin's adapter builds every catalog protocol for both planes through pi's public `createProvider`. Where a host's pi lacks something the vendor needs (Gemini 4 thinking levels on pi 1.1.0), that is a characterization row tagged `HOST_LIMITATION` with the channel and pi it applies to; live and mock cases that would exercise it skip by feature detection (pi's own exported predicate, never a version compare), a "Host limitations" list in the docs records it, and it is reported upstream. `test/architecture.test.ts` fails if `src` reads streamed tool-call deltas, partial arguments or thought signatures, so the desktop's patched pi stays invisible (D14''). P-T1a records this row and adds that tripwire. P-T1b gives the characterization rows one expectation per host cell where the cells differ (F10: no Go Google only on `rc`'s floor, pi `1.1.0`) and tags none of them `HOST_LIMITATION`; P-R1 adds the first. P-R1, the phase that removes the compensation, owns the rest: it deletes the wrapper and the restated defaults and rewrites the protocol order, and because the deletion is what exposes the limitation, it also adds the `HOST_LIMITATION` row (Gemini 4 thinking levels, channel `rc`, its floor pi `1.1.0`), the feature-detected skips in the mock case that inherits `google-thinking.test.ts`'s intents and in the live Gemini-level case (keyed on pi's own exported predicate answering for a Gemini 4 id, never on a version), the "Host limitations" list (in both READMEs, paired, or one doc they both link to), and the upstream report, recorded in its pull request. | +| D26 | **Host gaps are the host's job** (owner, 2026-11-03). The plugin never compensates for, re-implements or re-derives what the host's pi does, and never sniffs a channel or a version. So it deletes the Gemini thinking-level wrapper (`src/host/google-levels.ts`, which re-implemented pi 1.3's rule for an older pi), restates none of pi-ai's defaults (D17), and takes no base URL or protocol from pi's own catalog entries: the endpoint family table, the vendor's endpoints, is the only base source, and the protocol is the catalog's SDK, then the route's `api`, then completions (D16). It keeps what only the plugin can know (the gateway headers, the free-tier tool gate, the catalog's SDK to protocol, the endpoint families, per-plane credentials, the vendor's retirements) and what follows from owning the routes (the describe graft, listing declared ids). Google on the Go plane is routing, not a gap-fill: the plugin's adapter builds every catalog protocol for both planes through pi's public `createProvider`. Where a host's pi lacks something the vendor needs (Gemini 4 thinking levels on pi 1.1.0), that is a characterization row tagged `HOST_LIMITATION` with the channel and pi it applies to; live and mock cases that would exercise it skip by feature detection (pi's own exported predicate, never a version compare), a "Host limitations" list in the docs records it, and it is reported upstream. `test/architecture.test.ts` fails if `src` reads streamed tool-call deltas, partial arguments or thought signatures, so the desktop's patched pi stays invisible (D14''). P-T1a records this row and adds that tripwire. P-T1b gives the characterization rows one expectation per host cell where the cells differ (F10: no Go Google only on `rc`'s floor, pi `1.1.0`) and tags none of them `HOST_LIMITATION`; its binary e2e tags one host defect (alpha `0.2.2-alpha.2` under the cordis npm now resolves cannot save a config edit), and P-R1 adds the first pi row. P-R1, the phase that removes the compensation, owns the rest: it deletes the wrapper and the restated defaults and rewrites the protocol order, and because the deletion is what exposes the limitation, it also adds the `HOST_LIMITATION` row (Gemini 4 thinking levels, channel `rc`, its floor pi `1.1.0`), the feature-detected skips in the mock case that inherits `google-thinking.test.ts`'s intents and in the live Gemini-level case (keyed on pi's own exported predicate answering for a Gemini 4 id, never on a version), the "Host limitations" list (in both READMEs, paired, or one doc they both link to), and the upstream report, recorded in its pull request. | ## What the host supports (read from source) @@ -338,7 +338,7 @@ What stays private is pinned by contract tests against the real host packages: - `INTENDED_CHANGE`, right today and a phase changes it on purpose, for example: - completions models move from pi's own catalog base (legacy `/zen(/go)/v1` on `rc-floor`, the root `/inference(/go)/v1` on `alpha-floor`) to `/inference(/go)/openai/v1`, which is already `BASELINE` on `newest` (pi `1.5.0` files them there); - `decoratePlaneAdapters`' baseUrl repair never reaches the wire — a row of the `pi-ai-first` block only: in `plugin-first` the decoration never runs, and that row is a `KNOWN_DEFECT`. On `newest` the Zen plane's catalog base is the repaired base, so that one pair cannot tell the two apart and the case asserts it on the other routes and cells only. -- `HOST_LIMITATION` (D26): a host's pi lacks something the vendor needs, tagged with the channel and pi it applies to. It is recorded, skipped by feature detection where a case would exercise it, and reported upstream, never fixed in the plugin. No P-T1b row carries it (F10 is a per-cell fact the plugin's routing covers); P-R1 adds the first. +- `HOST_LIMITATION` (D26): a host's pi lacks something the vendor needs, tagged with the channel and pi it applies to. It is recorded, skipped by feature detection where a case would exercise it, and reported upstream, never fixed in the plugin. No in-process P-T1b row carries it (F10 is a per-cell fact the plugin's routing covers). The binary e2e carries one host defect under the same tag, because the plugin cannot fix it either: alpha `0.2.2-alpha.2` as npm installs it cannot save any config edit ("The `e2e-host` job" below). P-R1 adds the first pi row. 1. **P1a `fix:` the purge row's refusal and the endpoint copy.** The rest of what shipped broken in 1.4.1 was fixed on `main` (#133, #135) before this phase landed. - `settings-purge` shows the `message` of a refusal whose `error` is the host's `RemoteError` object. @@ -648,6 +648,8 @@ Two community plugins solve the same problem on the same host: [`FishBottle7/ope - (a) each channel's published binary, with pi as npm resolves it (D14''), on PRs that touch the host seam, nightly and on dispatch; it asserts the `newest` cell's rows; - (b) each channel's floor pi, nightly and on dispatch; it asserts that channel's floor cell's rows. +Neither cell pins cordis or the loader, which the published host declares as `~` ranges. A host that cannot run its own code under what npm resolved is a `[HOST_LIMITATION]` row, detected from the host's answer (a `TypeError` thrown by a `@deepseek-ai/dsh-*` package) and never from a version, and the rows that need the broken path skip (D26). Measured 2026-11-03: alpha `0.2.2-alpha.2` resolves cordis `4.0.6-alpha.2`, whose new `internal/update` call shape breaks `dsh-app-boot`'s own listener, so no config-editor save succeeds on either alpha cell and the save rows skip there; `dsh@alpha` has since moved to `0.2.2-alpha.3`, a channel bump the nightly drift report names (D22''). + ### The live gateway job (`OPENCODE_API_KEY` / `OPENCODE_GO_API_KEY`) - **Every PR:** the keyless matrix. A 404 means the path is gone; a 400 "Model is unavailable" or a 410 means the model is gone. It asserts that every cell has a model and was sent, and that no cell the plugin dispatches to with the default endpoint (`auto`, an alias of `modern`) answers 404, per host cell: on the rc floor, the desktop app's pi 1.1.0, legacy completions and modern Responses, Messages and Google (`TODAY_DISPATCH`); on both channels' npm install, pi 1.5.0, modern completions too (`NEXT_DISPATCH`), where P-R1 moves every host; the other legacy verdicts are P-R2's record. `endpoint: legacy` users dispatch Responses and Messages on both planes, and Zen's Google, to the legacy cells, which are recorded for P-R2, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell already asserted. diff --git a/test/e2e/host-binary.e2e.ts b/test/e2e/host-binary.e2e.ts index 55150ead..27b85a3f 100644 --- a/test/e2e/host-binary.e2e.ts +++ b/test/e2e/host-binary.e2e.ts @@ -39,6 +39,14 @@ * - `floor`: pi forced to the range's floor, the pi the desktop app ships. The * resolved pi must be that floor, and every row is the channel's floor cell. * + * Neither cell pins cordis or the loader: the published host declares them as + * `~` ranges, and npm resolves the newest patch, as it does for a user. Where + * that leaves the host unable to run its own code, the run records a + * `[HOST_LIMITATION]` row, detected from what the host answered and never from + * a version, and skips the rows that need the broken path (today: alpha's + * `0.2.2-alpha.2` under cordis `4.0.6-alpha.2`, where no config-editor save + * succeeds). + * * `DSH_HOST_PROBE_COPY` names a file to keep the probe's JSON in, for the CI * artifact; it is written whenever the probe wrote one, a failed run included. * `pnpm` must be on `PATH`, and `pnpm run build` must have run. @@ -98,6 +106,8 @@ const ROWS_PI = CELL === "floor" ? RECORD.piFloor : RECORD.pi; * `beforeAll`, false for an npm-resolved pi newer than the channel record. */ let rowsHold = false; +/** The cordis the prefix resolved, for the HOST_LIMITATION row's message. */ +let cordisVersion = ""; /** * Whether `version` satisfies a caret range `^x.y.z` (x >= 1): the same @@ -151,16 +161,41 @@ interface ProbeOutput { readonly callConfig: Record; readonly dispatch: Record; readonly describe: { user: string[]; value: string[] } | undefined; - readonly saveBlocked: { id: string; rejected?: string; accepted?: true }; - readonly saveCleared?: { - dropped: string[]; - rejected?: string; - accepted?: true; - }; - readonly save: { id: string; rejected?: string; accepted?: true }; + readonly saveBlocked: { id: string } & EditVerdict; + readonly saveCleared?: { dropped: string[] } & EditVerdict; + readonly save: { id: string } & EditVerdict; readonly afterSave?: unknown; } +/** + * One config-editor edit's answer. A rejection names the error's class and + * the package whose code threw it, which is how a failure inside the host's + * own code is told apart from a verdict pi-ai's checks reached. + */ +interface EditVerdict { + readonly accepted?: true; + readonly errorName?: string; + readonly rejected?: string; + readonly thrownBy?: string; +} + +/** + * Whether an edit failed with a TypeError thrown by the host's own cluster + * code (a `@deepseek-ai/dsh-*` package; cordis and its loader are excluded, + * since a plugin's bad listener can make them throw), rather than with a + * verdict pi-ai's checks reached: the host could not run its own save path. + * Measured on the alpha channel's npm install of `0.2.2-alpha.2`: its + * `dsh-app-boot` registers an `internal/update` listener in the old call shape + * (`(config, noSave, next)`), while the cordis its `~` range now resolves + * (`4.0.6-alpha.2`) calls the new one, so every config-editor save throws + * `next is not a function` there, before any plugin's code runs. Read from + * what the edit answered, never from a version, so a host that fixes it + * reads false the day it ships (D26). + */ +const hostListenerFailed = (verdict: EditVerdict | undefined): boolean => + verdict?.errorName === "TypeError" && + (verdict.thrownBy?.startsWith("@deepseek-ai/dsh-") ?? false); + /** The entries of a directory, or none when it cannot be read. */ const entriesOf = (dir: string): string[] => { try { @@ -304,6 +339,20 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { if (CELL === "floor") { expect(piVersion).toBe(RECORD.piFloor); } + cordisVersion = ( + JSON.parse( + readFileSync( + join( + PREFIX, + "node_modules", + "@deepseek-ai", + "cordis", + "package.json" + ), + "utf8" + ) + ) as { version: string } + ).version; rowsHold = piVersion === ROWS_PI; if (!rowsHold) { console.warn( @@ -537,7 +586,26 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { ]); }); - it("[KNOWN_DEFECT] a stored gap id blocks saving any new id on its plane", () => { + it("[HOST_LIMITATION] the host's own save path runs under the cordis npm resolved for it (it does not on alpha 0.2.2-alpha.2 as npm installs it today: every config-editor save throws inside the host's own code; reported upstream, D26)", () => { + if (!hostListenerFailed(out.saveBlocked)) { + return; + } + // Recorded, not fixed here: the host's own code cannot run, so no edit + // saves, whoever makes it. The two save rows below skip on this host. + // What is asserted is that the failure is that one and only that one: + // every edit the probe made failed the same way, inside the host's code. + console.warn( + `[e2e-host] ${CHANNEL} · ${CELL}: HOST_LIMITATION — the host's own ${String(out.saveBlocked.thrownBy)} throws under cordis ${cordisVersion} on every save (${String(out.saveBlocked.rejected)}); the save rows are skipped` + ); + expect(hostListenerFailed(out.saveCleared)).toBe(true); + expect(hostListenerFailed(out.save)).toBe(true); + expect(out.afterSave).toBeUndefined(); + }); + + it("[KNOWN_DEFECT] a stored gap id blocks saving any new id on its plane", (context) => { + if (hostListenerFailed(out.saveBlocked)) { + context.skip(); + } // pi-ai's strict check names the FIRST stored id the installed pi cannot // describe. Every recorded cell's pi lacks gpt-5.3-codex-spark, so it is // named; a newer pi may have learned it, and then names the next @@ -547,7 +615,10 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { ); }); - it("[KNOWN_DEFECT] a catalog-unknown id cannot be saved, once the plane stores no gap id", () => { + it("[KNOWN_DEFECT] a catalog-unknown id cannot be saved, once the plane stores no gap id", (context) => { + if (hostListenerFailed(out.saveBlocked)) { + context.skip(); + } // Clearing only ZEN_UNDESCRIBED assumes the installed pi still describes // the other opencode ids the profile declares. A pi newer than the record // asserts it too: one that retires one of them would fail here, and that diff --git a/test/e2e/host/probe-plugin.mjs b/test/e2e/host/probe-plugin.mjs index 163a07e3..5e761162 100644 --- a/test/e2e/host/probe-plugin.mjs +++ b/test/e2e/host/probe-plugin.mjs @@ -25,7 +25,8 @@ * onto the plane as booted (`saveBlocked`), then — after the ids in * `PROBE_SAVE_DROP` are removed from the plane (`saveCleared`) — once more * (`save`), so the last answer is about the id itself and not about a gap id - * the plane already stores; + * the plane already stores. A rejection also names the error's class and + * the PACKAGE whose code threw it (never a path); * - `PROBE_SAVE_DROP`: `["", …]`, the ids to remove first; * - `PROBE_ROUTES`: `["", …]`, the hidden routes to wait for, which the * e2e passes from `INTERNAL_ROUTES` so the probe holds no copy of them; @@ -52,6 +53,23 @@ const MESSAGE_LIMIT = 500; const messageOf = (error) => String(error?.message ?? error).slice(0, MESSAGE_LIMIT); +/** + * The package whose code threw `error`: the package of the top stack frame, + * read after its LAST `node_modules/`, or `"(not a package)"` when that frame + * is outside one, or `"(no stack)"` without a stack. Only the name is written, + * never the path, which would name the runner's directories. + */ +const thrownByOf = (error) => { + const frame = String(error?.stack ?? "") + .split("\n") + .find((line) => /^\s+at /u.test(line)); + if (frame === undefined) return "(no stack)"; + const parts = frame.replaceAll("\\", "/").split("node_modules/"); + if (parts.length < 2) return "(not a package)"; + const [first, second] = (parts.at(-1) ?? "").split("/"); + return first?.startsWith("@") ? `${first}/${second}` : first; +}; + /** A service the measurement needs, or a named failure where it is missing. */ const required = (ctx, service) => { const value = ctx.get(service); @@ -115,6 +133,8 @@ const settle = async (run) => { error: { code: typeof error?.code === "string" ? error.code : undefined, message: messageOf(error), + name: typeof error?.name === "string" ? error.name : undefined, + thrownBy: thrownByOf(error), }, }; } @@ -186,9 +206,16 @@ const editZen = async (editor, entry, mutate) => { return { ...current, providers }; }) ); + // `errorName` and `thrownBy` let the e2e tell the host's own failure (a + // TypeError inside the host's own code) from a verdict pi-ai's checks + // reached. return saved.error === undefined ? { accepted: true } - : { rejected: saved.error.message }; + : { + errorName: saved.error.name, + rejected: saved.error.message, + thrownBy: saved.error.thrownBy, + }; }; /** From 86880468447338cb3d25302dd54774fead72ee95 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 00:17:36 +0000 Subject: [PATCH 09/11] fix: pin the e2e-host floor cell to the shipped tree and narrow its host-limitation check The review of the P-T1b restack upheld fifteen findings; each outcome: - both-kept-1-F1 (fixed): the floor cell is now the tree the desktop app ships. The channel records name the whole cordis family (cordis, the loader, the include/group/timer plugins, cosmokit, schemastery), and `floorOverrides` forces it with pi at the range floor through npm `overrides`. The report step and the e2e assert every member on the floor cell; the npm cell stays fully npm-resolved. harness-channel holds each lockfile to the record's family (timer is the one member no lockfile resolves; the floor cell asserts it). Measured: with the pinned family the alpha floor saves, so the limitation is npm-only. - resolutions-1-T1B-R3 (fixed, with F1): the floor cache key is the channel, its dsh and a hash of the overrides, so a fill never freezes cordis. - both-kept-1-F2 (fixed): the detector matches only a TypeError thrown by dsh-app-boot, and HOST_LIMITATION needs a control to fail the same way: the probe now also edits its own entry, which no plugin listener reads. A mutant whose control saves while the plane edits fail turns both save rows red on the alpha npm cell. - resolutions-1-T1B-R2 (fixed): the row is titled as the limitation (channel, dsh, cell), skips where the host saves, and a BASELINE row asserts the control is accepted there. - both-kept-1-F3, resolutions-1-T1B-R1, rules-docs-1-PT1B-DOCS-1 (fixed in one edit): release step 3, D22'' and P-T3's two rows say a bump is data only while tiers A/B and the binary e2e stay green; one that turns them red re-measures the newest tables or adapts the tier listeners in its own reviewed pull request. The cordis alpha.1 to alpha.2 example is replaced, and the alpha.3 bump is named as the first such case. - both-kept-1-F4, rules-docs-1-PT1B-DOCS-3 (fixed): plan step 5 says no in-process row carries the tag and the binary e2e records one host-code defect; RowTag in host.ts holds the one definition, and the plan's tags bullet, CONTRIBUTING and host-characterization point to it. - both-kept-1-F5, rules-docs-1-PT1B-DOCS-2 (fixed): the two e2e-host bullets are rewritten, with no duplicate and no "always run"; the local floor snippet writes the record's overrides. - resolutions-1-T1B-R4 (fixed): the P-T1b restack row describes what landed: per-cell rows, pi-ai's defaults per channel, INVALID_CONFIG for the unknown id on every cell. - rules-docs-1-PT1B-DOCS-4 (fixed): the F10 row and the test list say per host cell. - rules-docs-1-PT1B-DOCS-5 (fixed): CONTRIBUTING and WHOLE_CATALOG's doc state the measured net change. - rules-docs-1-PT1B-DOCS-6 (fixed): the restack's comments name the ids by role or constant. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- .github/workflows/ci.yml | 67 ++++++++------ CONTRIBUTING.md | 19 ++-- docs/improvement-plan.md | 22 ++--- test/e2e/host-binary.e2e.ts | 143 +++++++++++++++++------------ test/e2e/host/probe-plugin.mjs | 80 ++++++++++++---- test/harness-channel.test.ts | 23 +++++ test/host-characterization.test.ts | 9 +- test/host-shapes.test.ts | 9 +- test/support/host-versions.ts | 65 ++++++++++++- test/support/host.ts | 32 ++++--- 10 files changed, 324 insertions(+), 145 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f20e8e5d..70b6c5f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -397,13 +397,17 @@ jobs: # One run per harness channel (D14''), each the channel's published binary # (`dsh@` from `test/support/host-versions.ts`) installed with npm the # way a user installs it, pi unpatched in both cells: - # - `npm`: pi as npm resolves it for the published host. The record's - # newest pi asserts every row; a pi published since the record skips the - # rows that depend on which ids pi describes, with a warning (the nightly - # `host-drift` job names it). Never cached: re-resolving is the point. - # - `floor`: pi forced to the bridge range's floor with npm `overrides`, the - # pi the desktop app ships (patched there, not here; the patch touches - # nothing the plugin reads). Cached by channel, dsh and pi. + # - `npm`: the published host exactly as npm resolves it, cordis and pi + # included. The record's newest pi asserts every row; a pi published since + # the record skips the rows that depend on which ids pi describes, with a + # warning (the nightly `host-drift` job names it). Never cached: + # re-resolving is the point. + # - `floor`: the tree the desktop app ships, forced with npm `overrides` + # (`floorOverrides` in host-versions.ts): the channel's whole cordis family + # at the record's versions and pi at the bridge range's floor (patched + # there, not here; the patch touches nothing the plugin reads). Cached by + # the channel, its dsh and a hash of those overrides, so a cache filled + # before the record moved is never reused. # Pull requests run the `npm` cell of both channels, and only when they touch # the host seam; dispatch and the nightly run add both `floor` cells. # @@ -481,41 +485,42 @@ jobs: id: versions if: steps.scope.outputs.run == 'true' run: | - node --experimental-strip-types --input-type=module -e "import { CHANNELS } from './test/support/host-versions.ts'; const r = CHANNELS[process.env.DSH_CHANNEL]; console.log('dsh=' + r.dsh); console.log('pi=' + r.pi); console.log('piFloor=' + r.piFloor); console.log('piRange=' + r.piRange)" >> "$GITHUB_OUTPUT" + node --experimental-strip-types --input-type=module -e "import { createHash } from 'node:crypto'; import { CHANNELS, floorOverrides } from './test/support/host-versions.ts'; const r = CHANNELS[process.env.DSH_CHANNEL]; const overrides = JSON.stringify(floorOverrides(r)); console.log('dsh=' + r.dsh); console.log('pi=' + r.pi); console.log('piRange=' + r.piRange); console.log('floorOverrides=' + overrides); console.log('floorKey=' + createHash('sha256').update(overrides).digest('hex').slice(0, 16))" >> "$GITHUB_OUTPUT" echo "PREFIX=$RUNNER_TEMP/dsh-host" >> "$GITHUB_ENV" - # The floor install forces one version, so it is cached by the channel, - # its dsh and that pi (the rest of the published host's tree is whatever - # its ranges resolved on the miss that filled the cache); the npm one is - # the point of re-resolving, so never. + # The floor install forces every version that matters (the record's + # cordis family and the floor pi), so it is cached by the channel, its + # dsh and a hash of those overrides; the npm one is the point of + # re-resolving, so never. - name: cache the floor host install id: host-cache if: steps.scope.outputs.run == 'true' && matrix.cell == 'floor' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/dsh-host - key: dsh-host-floor-${{ matrix.channel }}-${{ steps.versions.outputs.dsh }}-${{ steps.versions.outputs.piFloor }} + key: dsh-host-floor-${{ matrix.channel }}-${{ steps.versions.outputs.dsh }}-${{ steps.versions.outputs.floorKey }} - name: install the published host into a throwaway prefix if: steps.scope.outputs.run == 'true' && steps.host-cache.outputs.cache-hit != 'true' env: CELL: ${{ matrix.cell }} DSH: ${{ steps.versions.outputs.dsh }} - PI_FLOOR: ${{ steps.versions.outputs.piFloor }} + FLOOR_OVERRIDES: ${{ steps.versions.outputs.floorOverrides }} run: | mkdir -p "$PREFIX" - # The override must be in the prefix's own package.json BEFORE the - # install, or npm resolves the bridge's ^range to the newest pi. + # The overrides must be in the prefix's own package.json BEFORE the + # install, or npm resolves the bridge's ^range to the newest pi and + # the host's ~ ranges to the newest cordis patch. if [ "$CELL" = floor ]; then - printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s" } }\n' "$PI_FLOOR" > "$PREFIX/package.json" + node -e "process.stdout.write(JSON.stringify({ private: true, overrides: JSON.parse(process.env.FLOOR_OVERRIDES) }) + '\n')" > "$PREFIX/package.json" else printf '{ "private": true }\n' > "$PREFIX/package.json" fi npm install --prefix "$PREFIX" --no-audit --no-fund "@deepseek-ai/dsh@$DSH" - - name: report the pi, cordis and loader the host resolved + - name: report the pi and the cordis family the host resolved if: steps.scope.outputs.run == 'true' env: CELL: ${{ matrix.cell }} PI: ${{ steps.versions.outputs.pi }} - PI_FLOOR: ${{ steps.versions.outputs.piFloor }} + FLOOR_OVERRIDES: ${{ steps.versions.outputs.floorOverrides }} run: | resolved="$(node -p "require(process.env.PREFIX + '/node_modules/@earendil-works/pi-ai/package.json').version")" echo "${DSH_CHANNEL} · ${CELL}: pi $resolved" @@ -524,19 +529,25 @@ jobs: echo "::error::expected one copy of pi under the host prefix, found $copies" exit 1 fi - if [ "$CELL" = floor ] && [ "$resolved" != "$PI_FLOOR" ]; then - echo "::error::the floor cell resolved pi $resolved, not $PI_FLOOR" - exit 1 - fi if [ "$CELL" = npm ] && [ "$resolved" != "$PI" ]; then echo "::warning::npm resolved pi $resolved for ${DSH_CHANNEL}, not the record's $PI: the rows that depend on which ids pi describes are skipped" fi - # Reported, not asserted: the published host declares cordis and the - # loader as tilde ranges, and a new patch there is the channel's - # release, which the nightly drift report names. - for name in @deepseek-ai/cordis @deepseek-ai/cordis-plugin-loader; do - echo "${DSH_CHANNEL} · ${CELL}: $name $(node -p "require(process.env.PREFIX + '/node_modules/$name/package.json').version")" + # Every package the floor cell forces (pi and the cordis family): the + # floor cell must hold exactly the record's version, a cached prefix + # included; the npm cell reports what npm resolved, which is the + # channel's release as a user gets it (the nightly drift report + # names a move past the record). + failed=0 + for name in $(node -p "Object.keys(JSON.parse(process.env.FLOOR_OVERRIDES)).join(' ')"); do + want="$(node -p "JSON.parse(process.env.FLOOR_OVERRIDES)['$name']")" + got="$(node -p "require(process.env.PREFIX + '/node_modules/$name/package.json').version")" + echo "${DSH_CHANNEL} · ${CELL}: $name $got" + if [ "$CELL" = floor ] && [ "$got" != "$want" ]; then + echo "::error::the floor cell resolved $name $got, not the record's $want" + failed=1 + fi done + exit "$failed" - if: steps.scope.outputs.run == 'true' run: pnpm run build - name: boot the host binary with the probe diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 555bcb15..dbd498eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ The e2e files, one job each: `test/support/host.ts` builds the host from the packages this repository installs: the harness channel `test/support/host-versions.ts` selects (the `@deepseek-ai` cluster at that channel's published tag), with pi unpatched at the newest version the bridge's range admits, or at the range's floor after `scripts/use-channel.ts --floor`. The routing work is judged against it: the contract tests pin the host facts the design rests on that the host can answer in-process (`docs/improvement-plan.md`: F1, F2, F4, F5, F8, F10–F16, F18 and F20, each by number; F6 and F19's auth half (constructing a `PiAiAdapter`) are first exercised by P-R1; F19's export half is pinned in `host-shapes` (`recordKeyFor`); F17 is honoured, not pinned — tier B fixes the apply order and the binary e2e infers it; F9 is a fact about opencode's own client, outside this host), and the characterization rows record what today's plugin does. -**Host cells.** The rows are measured on three cells (`HostCell` in `host.ts`): `newest`, either channel's default install (pi 1.5.0 today, one pi for both channels, so one column; a fixture case fails the day the two records disagree), and `rc-floor` (pi 1.1.0) and `alpha-floor` (pi 1.3.0), the pis the desktop app ships. Where the cells differ, a row carries one expectation per cell (`forHost`, keyed by the cell's role, never by a pi version) or per channel (`forChannel`, for what the `@deepseek-ai` cluster decides, such as pi-ai's own defaults), and every tier file runs on whichever cell is installed. CI runs the unit suite, tiers A and B included, on both channels' `newest` install on every pull request and on both floors nightly and on dispatch; locally, `scripts/use-channel.ts` switches the install and the same run reads the cell from the marker it leaves. The cells differ today in pi's catalog bases (the legacy family on 1.1.0, the root `/inference(/go)/v1` on 1.3.0, per-protocol roots on 1.5.0, which also dropped five Zen models), in which S0 ids pi can place (`PI_GAPS_BY_CELL`: 1.5.0 places both Go gap ids), in whether pi's Go provider carries Google (F10: not on 1.1.0), and in pi-ai's own defaults (the alpha cluster raised them to 1_000_000 / 128_000). +**Host cells.** The rows are measured on three cells (`HostCell` in `host.ts`): `newest`, either channel's default install (pi 1.5.0 today, one pi for both channels, so one column; a fixture case fails the day the two records disagree), and `rc-floor` (pi 1.1.0) and `alpha-floor` (pi 1.3.0), the pis the desktop app ships. Where the cells differ, a row carries one expectation per cell (`forHost`, keyed by the cell's role, never by a pi version) or per channel (`forChannel`, for what the `@deepseek-ai` cluster decides, such as pi-ai's own defaults), and every tier file runs on whichever cell is installed. CI runs the unit suite, tiers A and B included, on both channels' `newest` install on every pull request and on both floors nightly and on dispatch; locally, `scripts/use-channel.ts` switches the install and the same run reads the cell from the marker it leaves. The cells differ today in pi's catalog bases (the legacy family on 1.1.0, the root `/inference(/go)/v1` on 1.3.0, per-protocol roots on 1.5.0, whose Zen catalog is net five models smaller than 1.3.0's), in which S0 ids pi can place (`PI_GAPS_BY_CELL`: 1.5.0 places both Go gap ids), in whether pi's Go provider carries Google (F10: not on 1.1.0), and in pi-ai's own defaults (the alpha cluster raised them to 1_000_000 / 128_000). Two tiers, both in-process: @@ -69,7 +69,7 @@ What every case does: - **Give tier-B cases 30 s**, and wait with a deadline (`waitForRoutes`, 10 s by default, so a route that never registers fails by name inside the hook's budget), never a fixed sleep: the plugin mounts its hidden routes asynchronously. - **A first row that fails does not hang the boot.** Tier B holds the second of pi-ai and the plugin until the first applies OR its fiber settles in any state, so a row whose config the schema rejects boots with that fiber FAILED (`fiber.await()` rethrows the real error) and the other row applied. -**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one cell's host lacks or breaks, its pi or its own code, that no phase of this plugin fixes: the title names the channel and the host, and it is reported upstream, D26). A facet's tag is per cell: one that pi 1.5.0 already serves the way P-R1 keeps (completions on the per-protocol family, Go `gemini-4-argon` validating, Go `minimax-m3.1-flash` placed) is `[BASELINE]` on `newest` and keeps its floor tag on the floors. A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon`'s call config, on the floors only: pi 1.5.0 places the id in either order) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. +**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one cell's host lacks or breaks, its pi or its own code, that no phase of this plugin fixes: the title names the channel and the host, and it is reported upstream, D26). `RowTag` in `test/support/host.ts` holds the one definition of the four. A facet's tag is per cell: one that pi 1.5.0 already serves the way P-R1 keeps (completions on the per-protocol family, Go `gemini-4-argon` validating, Go `minimax-m3.1-flash` placed) is `[BASELINE]` on `newest` and keeps its floor tag on the floors. A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon`'s call config, on the floors only: pi 1.5.0 places the id in either order) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. **A catalog refresh can touch the rows.** `S0_TODAY`'s context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case compares each row's context window and effort ids to the bundled catalog's derivation and names the model that moved; when it fails on a refresh pull request, update the row in `S0_RC_FLOOR` (the context windows and efforts every cell inherits), and in a cell's `withMoved` entry where that cell restates the row. A phase that moves a non-`BASELINE` `info` facet off the catalog on purpose (the user's context window, a toggle model's Off/On) adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. @@ -79,15 +79,15 @@ What every case does: `.github/workflows/ci.yml`'s `e2e-host` job installs each channel's published `@deepseek-ai/dsh` (its record's `dsh`) into a throwaway prefix with npm, the way a user installs it, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the temporary home's credentials document (`$DSH_HOME/.credentials.yaml`, shared by every profile under that home), boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. -- **Cells, per channel, pi unpatched in both.** `npm` takes the pi npm resolves for the published host, what an npm user runs, and asserts the `newest` rows while that pi is the channel record's; a pi published since the record still has to sit in the bridge's range, but the rows that depend on which ids pi describes (the keyed listing and the per-model rows) are skipped with a warning, and the nightly `host-drift` job names the new pi. The save rows always run; they assume pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage that as a pi change first. `floor` forces pi to the range's floor with an npm `override` (the desktop app's pi), asserts it resolved, and asserts every row of the channel's floor cell. cordis and the loader are reported, not forced: the published host declares them as tilde ranges, and a new patch there is the channel's release. The installed `dsh` must be the channel record's, so a prefix of the wrong channel fails by name. -- **cordis and the loader are not pinned, and the host's own failures are recorded, not worked around.** The published host declares both as `~` ranges, so each cell gets the newest patch npm resolves, as a user does, and the job reports the versions. Where that leaves the host unable to run its own code, the run records a `[HOST_LIMITATION]` row and skips the rows that need the broken path, detecting it from what the host answered (a `TypeError` thrown by a `@deepseek-ai/dsh-*` package, which the probe names) and never from a version (D26). Today that is alpha's `0.2.2-alpha.2` under cordis `4.0.6-alpha.2`: its `dsh-app-boot` registers an `internal/update` listener in the old call shape, so every config-editor save throws `next is not a function` before any plugin runs, and the two save rows skip on both alpha cells. +- **Cells, per channel, pi unpatched in both.** `npm` is the published host exactly as npm resolves it, cordis and pi included, what an npm user runs, and asserts the `newest` rows while its pi is the channel record's; a pi published since the record still has to sit in the bridge's range, but the rows that depend on which ids pi describes (the keyed listing and the per-model rows) are skipped with a warning, and the nightly `host-drift` job names the new pi. `floor` is the tree the desktop app ships: npm `overrides` (`floorOverrides` in `host-versions.ts`) force the channel's whole cordis family (cordis, the loader, the include, group and timer plugins, cosmokit, schemastery) at the record's versions and pi at the range's floor; the job and the e2e assert each one, and the cell asserts every row of the channel's floor cell. It is cached by channel, `dsh` and a hash of those overrides, so it never runs a tree frozen when the cache was filled. The installed `dsh` must be the channel record's, so a prefix of the wrong channel fails by name. The save rows run whatever pi resolved (they do not depend on which ids pi describes, though they assume pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage that as a pi change first); they skip only where the host itself cannot save (next bullet). +- **The host's own failures are recorded, not worked around.** Only the `npm` cell lets cordis float, as the published host's `~` ranges do for a user, and the job reports what it resolved. Where that leaves the host unable to run its own save path, the run records a `[HOST_LIMITATION]` row (titled with the channel and the `dsh`) and skips the save rows. It is detected from what the host answered, never from a version (D26): the plane edit and a control edit no plugin's listener reads (the probe's own entry) both fail with the same `TypeError` thrown by `@deepseek-ai/dsh-app-boot`; where the host saves, the row skips and a `[BASELINE]` row asserts the control was accepted. Today that is alpha's `0.2.2-alpha.2` as npm installs it: cordis `4.0.6-alpha.2` calls `internal/update` in a new shape while `dsh-app-boot` registers its listener in the old one, so every config-editor save throws `next is not a function`. The alpha floor, on the cordis the harness vendors at the tag, saves. - **Triggers.** Dispatch and the nightly run take both cells of both channels. A pull request takes the `npm` cell of both channels, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` (what the job installs), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. To run it locally, install a channel's host into a prefix outside the checkout, then point the suite at it (placeholder paths; `pnpm` must be on `PATH`). The first line reads the channel's record from `test/support/host-versions.ts`, the same way the CI job does, so the snippet never names a version itself: ```sh export DSH_CHANNEL=alpha # or rc -eval "$(node --experimental-strip-types --input-type=module -e "import { CHANNELS } from './test/support/host-versions.ts'; const r = CHANNELS[process.env.DSH_CHANNEL]; console.log('DSH=' + r.dsh + ' PI_FLOOR=' + r.piFloor)")" +eval "$(node --experimental-strip-types --input-type=module -e "import { CHANNELS } from './test/support/host-versions.ts'; const r = CHANNELS[process.env.DSH_CHANNEL]; console.log('DSH=' + r.dsh)")" mkdir -p /home/you/dsh-host printf '{ "private": true }\n' > /home/you/dsh-host/package.json npm install --prefix /home/you/dsh-host "@deepseek-ai/dsh@$DSH" @@ -95,7 +95,12 @@ pnpm run build DSH_HOST_E2E=1 DSH_HOST_PREFIX=/home/you/dsh-host pnpm run test:e2e:host --run ``` -For the `floor` cell, use a fresh prefix whose `package.json` also carries `"overrides"` pinning `@earendil-works/pi-ai` to `$PI_FLOOR` before the install, and run the suite with `DSH_HOST_PI_CELL=floor`. +For the `floor` cell, use a fresh prefix whose `package.json` carries the channel's `floorOverrides` (the cordis family at the record's versions and pi at the range's floor) as `"overrides"` before the install, written the same way the CI job writes it, and run the suite with `DSH_HOST_PI_CELL=floor`: + +```sh +mkdir -p /home/you/dsh-host-floor +node --experimental-strip-types --input-type=module -e "import { CHANNELS, floorOverrides } from './test/support/host-versions.ts'; console.log(JSON.stringify({ private: true, overrides: floorOverrides(CHANNELS[process.env.DSH_CHANNEL]) }))" > /home/you/dsh-host-floor/package.json +``` ## Live profile loop @@ -156,7 +161,7 @@ git diff --name-only main... \ 1. Use conventional commits (`feat:`, `fix:`) — release-please opens the version-bump + changelog PR automatically; merging it tags and creates the GitHub Release, which fires OIDC publishing to npm and GitHub Packages. 2. Avoid manual releases: release-please anchors on the latest GitHub **Release**, not the latest tag, so a bare tag is invisible to it (`AGENTS.md`, "Releasing"). If one is unavoidable, bump `version` in `package.json`, commit, tag, push the tag, and create the GitHub Release in the same step (`gh release create vX.Y.Z --generate-notes`). Never touch `CHANGELOG.md`: release-please owns it. -3. On every release, after smoke-testing (registry install resolves, plugin loads in the DSH Web profile, and a free-tier Zen call succeeds), check the **Supported: the latest release of both DeepSeek Harness channels** line and the **Support policy** paragraph under "Compatibility & Verification" in both READMEs. Their "today" versions, pi ranges and floors, and the default install named a paragraph below, are copied from the channel records in `test/support/host-versions.ts`, and `test/harness-channel.test.ts` holds them to the records. A channel bump **inside the prerelease tuples the peer ranges already admit** (`0.2.2-alpha.2` to `0.2.2-alpha.3`, cordis `4.0.6-alpha.1` to `4.0.6-alpha.2`) is version data only, never `src` or test logic, and this is its whole set: for `rc`, the rc record and `channels/rc/pnpm-lock.yaml` (`scripts/use-channel.ts rc --relock`); for `alpha`, the alpha record, `package.json`'s exact `@deepseek-ai/*` pins and the root lockfile (`pnpm install`); for either, those README sentences in both languages, and `minimumReleaseAgeExclude` lines in `pnpm-workspace.yaml` for the exact versions when the cut is younger than pnpm's one-day release-age window. A pi move is the same data: `pnpm install` and `--relock` keep the pi a lockfile already satisfies, so move it with `pnpm update @earendil-works/pi-ai --no-save` (the root lockfile; `package.json` keeps the union range), then `scripts/use-channel.ts rc --relock`, which carries the root's pi into the rc lockfile, and set each record's `pi`. The same update is what leaves one pi when a new bridge raises its floor above the locked one. The F10 characterization (`test/mount-protocols.test.ts`) is keyed by pi line, so a pi inside a line needs no test change; a pi outside every line is a new major, and a reviewed change. A cut that **opens a new prerelease tuple** (dsh `0.2.3-*`, cordis `4.0.7-*`) is not data: semver admits a prerelease only through a comparator in its own tuple, so it is a reviewed range change that also adds the matching `>=X.Y.Z-0 =X.Y.Z-0 ; readonly describe: { user: string[]; value: string[] } | undefined; readonly saveBlocked: { id: string } & EditVerdict; + /** The control: an edit of the probe's own entry, made after the saves. */ + readonly control?: EditVerdict; readonly saveCleared?: { dropped: string[] } & EditVerdict; readonly save: { id: string } & EditVerdict; readonly afterSave?: unknown; @@ -180,21 +188,30 @@ interface EditVerdict { } /** - * Whether an edit failed with a TypeError thrown by the host's own cluster - * code (a `@deepseek-ai/dsh-*` package; cordis and its loader are excluded, - * since a plugin's bad listener can make them throw), rather than with a - * verdict pi-ai's checks reached: the host could not run its own save path. - * Measured on the alpha channel's npm install of `0.2.2-alpha.2`: its - * `dsh-app-boot` registers an `internal/update` listener in the old call shape - * (`(config, noSave, next)`), while the cordis its `~` range now resolves - * (`4.0.6-alpha.2`) calls the new one, so every config-editor save throws - * `next is not a function` there, before any plugin's code runs. Read from - * what the edit answered, never from a version, so a host that fixes it - * reads false the day it ships (D26). + * Whether an edit failed the way the host's own save path fails: a TypeError + * thrown by `@deepseek-ai/dsh-app-boot`, a package this plugin registers no + * listener on. Measured on the alpha channel's npm install of + * `0.2.2-alpha.2`: its `dsh-app-boot` registers an `internal/update` listener + * in the old call shape (`(config, noSave, next)`), while the cordis its `~` + * range now resolves (`4.0.6-alpha.2`) calls the new one, so the save throws + * `next is not a function` there. */ const hostListenerFailed = (verdict: EditVerdict | undefined): boolean => verdict?.errorName === "TypeError" && - (verdict.thrownBy?.startsWith("@deepseek-ai/dsh-") ?? false); + verdict.thrownBy === "@deepseek-ai/dsh-app-boot"; + +/** + * Whether the host cannot save ANY config edit, whoever makes it: the plane + * edit failed in the host's own code, and so did the control — an edit of the + * probe's own entry, which no plugin's listener reads — with the same error. + * A failure of the plane edit alone is about the plane, and the save rows + * report it. Read from what the edits answered, never from a version, so a + * host that fixes it reads false the day it ships (D26). + */ +const hostCannotSave = (probe: ProbeOutput): boolean => + hostListenerFailed(probe.saveBlocked) && + hostListenerFailed(probe.control) && + probe.control?.rejected === probe.saveBlocked.rejected; /** The entries of a directory, or none when it cannot be read. */ const entriesOf = (dir: string): string[] => { @@ -336,23 +353,24 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { satisfiesCaret(piVersion, RECORD.piRange), `pi ${piVersion} is outside the bridge's ${RECORD.piRange}` ).toBe(true); + const installedVersion = (name: string): string => + ( + JSON.parse( + readFileSync( + join(PREFIX, "node_modules", ...name.split("/"), "package.json"), + "utf8" + ) + ) as { version: string } + ).version; if (CELL === "floor") { expect(piVersion).toBe(RECORD.piFloor); + // The floor cell is the tree the desktop app ships, so the cordis family + // is the record's, not whatever the host's `~` ranges resolve today. + for (const [name, version] of Object.entries(cordisFamily(RECORD))) { + expect(installedVersion(name), name).toBe(version); + } } - cordisVersion = ( - JSON.parse( - readFileSync( - join( - PREFIX, - "node_modules", - "@deepseek-ai", - "cordis", - "package.json" - ), - "utf8" - ) - ) as { version: string } - ).version; + cordisVersion = installedVersion("@deepseek-ai/cordis"); rowsHold = piVersion === ROWS_PI; if (!rowsHold) { console.warn( @@ -586,29 +604,40 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { ]); }); - it("[HOST_LIMITATION] the host's own save path runs under the cordis npm resolved for it (it does not on alpha 0.2.2-alpha.2 as npm installs it today: every config-editor save throws inside the host's own code; reported upstream, D26)", () => { - if (!hostListenerFailed(out.saveBlocked)) { - return; + it(`[HOST_LIMITATION] ${CHANNEL} ${RECORD.dsh} (${CELL} cell): no config-editor save succeeds, because the host's own code throws (reported upstream, D26)`, (context) => { + if (!hostCannotSave(out)) { + // This host saves (or fails only the plane edit, which the save rows + // report): the limitation is absent here, so there is nothing to record. + context.skip(); } // Recorded, not fixed here: the host's own code cannot run, so no edit - // saves, whoever makes it. The two save rows below skip on this host. - // What is asserted is that the failure is that one and only that one: - // every edit the probe made failed the same way, inside the host's code. + // saves, whoever makes it. The save rows below skip on this host. What is + // asserted is that the failure is that one and only that one: every edit + // the probe made failed the same way, inside the host's code. console.warn( - `[e2e-host] ${CHANNEL} · ${CELL}: HOST_LIMITATION — the host's own ${String(out.saveBlocked.thrownBy)} throws under cordis ${cordisVersion} on every save (${String(out.saveBlocked.rejected)}); the save rows are skipped` + `[e2e-host] ${CHANNEL} · ${CELL}: HOST_LIMITATION — the host's own ${String(out.saveBlocked.thrownBy)} throws under cordis ${cordisVersion} on every save, the control included (${String(out.saveBlocked.rejected)}); the save rows are skipped` ); expect(hostListenerFailed(out.saveCleared)).toBe(true); expect(hostListenerFailed(out.save)).toBe(true); expect(out.afterSave).toBeUndefined(); }); + it("[BASELINE] the control edit (the probe's own entry, which no plugin reads) saves", (context) => { + if (hostCannotSave(out)) { + context.skip(); + } + // The known answer the HOST_LIMITATION detector rests on: where the host + // can save, an edit the plugin cannot reach is accepted. + expect(out.control).toEqual({ accepted: true }); + }); + it("[KNOWN_DEFECT] a stored gap id blocks saving any new id on its plane", (context) => { - if (hostListenerFailed(out.saveBlocked)) { + if (hostCannotSave(out)) { context.skip(); } // pi-ai's strict check names the FIRST stored id the installed pi cannot - // describe. Every recorded cell's pi lacks gpt-5.3-codex-spark, so it is - // named; a newer pi may have learned it, and then names the next + // describe. Every recorded cell's pi lacks the first of ZEN_UNDESCRIBED, + // so it is named; a newer pi may have learned it, and then names the next // undescribed id — so that run asserts only that one of them blocks it. expect(out.saveBlocked.rejected).toMatch( rowsHold ? ROWS.saveBlockedByStoredGap : SAVE_BLOCKED_BY_ANY_UNDESCRIBED @@ -616,7 +645,7 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { }); it("[KNOWN_DEFECT] a catalog-unknown id cannot be saved, once the plane stores no gap id", (context) => { - if (hostListenerFailed(out.saveBlocked)) { + if (hostCannotSave(out)) { context.skip(); } // Clearing only ZEN_UNDESCRIBED assumes the installed pi still describes diff --git a/test/e2e/host/probe-plugin.mjs b/test/e2e/host/probe-plugin.mjs index 5e761162..8a54f26b 100644 --- a/test/e2e/host/probe-plugin.mjs +++ b/test/e2e/host/probe-plugin.mjs @@ -28,6 +28,9 @@ * the plane already stores. A rejection also names the error's class and * the PACKAGE whose code threw it (never a path); * - `PROBE_SAVE_DROP`: `["", …]`, the ids to remove first; + * after those, one CONTROL edit (`control`) of the probe's OWN entry, which + * no plugin's listener reads, so a failure there is the host's save path + * failing whoever edits, not a verdict about the plane; * - `PROBE_ROUTES`: `["", …]`, the hidden routes to wait for, which the * e2e passes from `INTERNAL_ROUTES` so the probe holds no copy of them; * - `PROBE_PI_PACKAGE`: the package name pi-ai's loader entry carries; @@ -197,15 +200,9 @@ const askAbout = async (llm, ids) => { return { callConfig, listModels, modelInfo }; }; -/** One config-editor edit of `opencode`, as the Settings page makes it. */ -const editZen = async (editor, entry, mutate) => { - const saved = await settle(() => - editor.edit(entry, (current) => { - const providers = { ...current.providers }; - providers.opencode = mutate(providers.opencode ?? {}); - return { ...current, providers }; - }) - ); +/** One config-editor edit of `entry`, as the Settings page makes it. */ +const editEntry = async (editor, entry, change) => { + const saved = await settle(() => editor.edit(entry, change)); // `errorName` and `thrownBy` let the e2e tell the host's own failure (a // TypeError inside the host's own code) from a verdict pi-ai's checks // reached. @@ -218,6 +215,34 @@ const editZen = async (editor, entry, mutate) => { }; }; +/** One config-editor edit of `opencode` in pi-ai's entry. */ +const editZen = (editor, entry, mutate) => + editEntry(editor, entry, (current) => { + const providers = { ...current.providers }; + providers.opencode = mutate(providers.opencode ?? {}); + return { ...current, providers }; + }); + +/** + * The control: one edit of the probe's own entry, which no plugin's listener + * reads. The host restarts the probe's fiber when it applies, which + * `apply` tells apart from a real stop (`state.controlling`). + */ +const controlEdit = async (editor, state) => { + const entry = editor + .entries() + .find((candidate) => candidate.options?.name === name); + if (entry === undefined) { + return { + errorName: "Error", + rejected: "probe: the config editor lists no entry of the probe's own", + thrownBy: "(probe)", + }; + } + state.controlling = true; + return editEntry(editor, entry, (current) => ({ ...current, control: 1 })); +}; + /** * Save `saveId` into `opencode`'s models: onto the plane as booted, then onto * the plane without the `drop` ids, so the last verdict is about `saveId`. @@ -259,7 +284,7 @@ const saveSequence = async (llm, editor, saveId, drop) => { }; }; -const measure = async (ctx) => { +const measure = async (ctx, state) => { if (HIDDEN_ROUTES.length === 0 || PI_AI_PACKAGE.length === 0) { throw new Error("probe: PROBE_ROUTES and PROBE_PI_PACKAGE must be set"); } @@ -333,33 +358,50 @@ const measure = async (ctx) => { const drop = JSON.parse(process.env.PROBE_SAVE_DROP ?? "[]"); const saving = saveId.length === 0 ? {} : await saveSequence(llm, editor, saveId, drop); + const describe = + piDescriptor === undefined + ? undefined + : { + user: Object.keys(piDescriptor.user?.providers ?? {}), + value: Object.keys(piDescriptor.value?.providers ?? {}), + }; + // Last: a control that saves restarts this fiber. + const control = + saveId.length === 0 ? undefined : await controlEdit(editor, state); return { ...registered, ...asked, ...saving, - describe: - piDescriptor === undefined - ? undefined - : { - user: Object.keys(piDescriptor.user?.providers ?? {}), - value: Object.keys(piDescriptor.value?.providers ?? {}), - }, + control, + describe, discovery: discovered.ok ? { count: discovered.ok.length } : discovered, dispatch, listModelsAfter, }; }; +/** + * Whether a measurement has started in this process: the control edit + * restarts the probe's fiber, and the restarted `apply` must not measure + * again (the module, and so this flag, outlives the fiber). + */ +let started = false; + export const apply = (ctx) => { + if (started) return; + started = true; let stopped = false; + const state = { controlling: false }; ctx.effect?.(() => () => { - stopped = true; + // The control edit's own restart is not a stop: its answer is still + // written. + if (!state.controlling) stopped = true; }); // Not awaited: `apply` returns at once, and the measurement runs once the // loader has settled every row. void (async () => { try { - const out = await measure(ctx); + const out = await measure(ctx, state); if (stopped) return; writeFileSync(OUT, `${JSON.stringify(out, null, 2)}\n`); setTimeout(() => process.exit(0), 200); diff --git a/test/harness-channel.test.ts b/test/harness-channel.test.ts index 448d0703..c306cbc6 100644 --- a/test/harness-channel.test.ts +++ b/test/harness-channel.test.ts @@ -44,7 +44,9 @@ import { channelManifest, pinFor } from "../scripts/channel-manifest.ts"; import { type Channel, CHANNELS, + cordisFamily, DEFAULT_CHANNEL, + floorOverrides, PI_UNION_RANGE, } from "./support/host-versions.ts"; @@ -467,6 +469,27 @@ describe.each(Object.keys(CHANNELS) as Channel[])( ]); }); + it("resolves the cordis family the record names, the e2e-host floor cell's overrides", () => { + // `cordisFamily` is what the `e2e-host` job forces on the floor cell, so + // a family member here and the record disagreeing would make that cell + // run a tree this install never measured. The timer plugin is the one + // member nothing here depends on: no lockfile resolves it, and the floor + // cell asserts it against the published host's tree instead. + const lock = lockfile(record.lockfile); + const family = Object.entries(cordisFamily(record)); + expect(family).toHaveLength(7); + const timer = "@deepseek-ai/cordis-plugin-timer"; + expect(lock.versionsOf(timer), timer).toEqual([]); + const resolved = family.filter(([name]) => name !== timer); + for (const [name, version] of resolved) { + expect(lock.versionsOf(name), name).toEqual([version]); + } + expect(floorOverrides(record)).toStrictEqual({ + ...Object.fromEntries(family), + "@earendil-works/pi-ai": record.piFloor, + }); + }); + it("holds no harness package on the other channel's prerelease line", () => { // The checks above name dsh, cordis, the loader and schemastery; a // relock that kept the other channel's optional peers (`-include`, diff --git a/test/host-characterization.test.ts b/test/host-characterization.test.ts index 47e03ad7..cbaef5e9 100644 --- a/test/host-characterization.test.ts +++ b/test/host-characterization.test.ts @@ -13,9 +13,10 @@ * - `[KNOWN_DEFECT]` is wrong today, and a phase fixes it; * - `[INTENDED_CHANGE]` is right today, and a phase changes it on purpose; * - `[BASELINE]` must survive every phase; - * - `[HOST_LIMITATION]` is something one cell's pi lacks that the vendor's - * service needs, named with its channel and pi, reported upstream and never - * fixed here (D26). No row in this file carries it yet: P-R1 adds the first. + * - `[HOST_LIMITATION]` is something one cell's host lacks or breaks, named + * with its channel and host (the definition is `RowTag` in + * `support/host.ts`). No row in this file carries it yet: P-R1 adds the + * first pi row; the binary e2e records one host-code defect. * * Where the hosts differ, a row carries one expectation per cell * (`S0_TODAY_BY_CELL`, `forHost`), and a title names the cell's tag: a facet @@ -469,7 +470,7 @@ describe( ); } expect(recorded).toMatch(SAVE_BLOCKED_BY_ANY_UNDESCRIBED); - // A newer pi that learned gpt-5.3-codex-spark names the next one. + // A newer pi that learned the first of ZEN_UNDESCRIBED names the next. expect(`provider "opencode" model "${UNKNOWN_ID}" needs an api`).toMatch( SAVE_BLOCKED_BY_ANY_UNDESCRIBED ); diff --git a/test/host-shapes.test.ts b/test/host-shapes.test.ts index a404d0ff..48c8b080 100644 --- a/test/host-shapes.test.ts +++ b/test/host-shapes.test.ts @@ -57,9 +57,12 @@ const ZEN_BASE = "https://opencode.ai"; /** * F1/F2, measured per cell: a model-less plane resolves pi's whole catalog. * The bases moved twice under the plugin: the legacy family on pi 1.1.0, the - * root `/inference(/go)/v1` on 1.3.0, and per-protocol roots on 1.5.0, which - * also dropped four Zen completions models and one Anthropic one. Go gained a - * Google model on 1.3.0 (F10). + * root `/inference(/go)/v1` on 1.3.0, and per-protocol roots on 1.5.0, where + * the per-protocol counts moved too (Zen, net of 1.3.0: four fewer + * completions models and one fewer Anthropic one). Go gained a Google model + * on 1.3.0 (F10). A pi bump whose catalog moved re-measures the `newest` + * column in its own pull request, as a reviewed change (CONTRIBUTING, release + * step 3). */ const WHOLE_CATALOG: Readonly< Record< diff --git a/test/support/host-versions.ts b/test/support/host-versions.ts index 55ac4fba..cf39d086 100644 --- a/test/support/host-versions.ts +++ b/test/support/host-versions.ts @@ -4,8 +4,11 @@ * * - `rc` is `@deepseek-ai/dsh@latest`; `alpha` is `@deepseek-ai/dsh@alpha`. * Each record names the `@deepseek-ai` cluster's tag, cordis and its loader - * (each on its own line, at what the cluster's `~` range resolves), and the - * `@deepseek-ai/schemastery` the cluster's range resolves. + * (each on its own line, at what the cluster's `~` range resolves), the + * `@deepseek-ai/schemastery` the cluster's range resolves, and the rest of + * the cordis family the published host installs (`cordisFamily`): the + * versions the harness vendors at the channel's tag, which is the tree the + * desktop app ships and the one the `e2e-host` job's floor cell forces. * - `piRange` is the range the bridge (`dsh-llm-pi-ai`) declares. Nothing in * the published packages narrows it: no shrinkwrap, no bundled copy, no * install script and no runtime version check, so an npm install runs pi @@ -40,10 +43,16 @@ export type Channel = "alpha" | "rc"; export interface ChannelRecord { /** `@deepseek-ai/cordis`, what the cluster's `~` range resolves. */ readonly cordis: string; + /** `@deepseek-ai/cosmokit`, cordis's own utility library. */ + readonly cosmokit: string; /** The npm dist-tag the channel publishes `@deepseek-ai/dsh` under. */ readonly distTag: "alpha" | "latest"; /** The `@deepseek-ai/dsh-*` cluster's version. */ readonly dsh: string; + /** `@deepseek-ai/cordis-plugin-group`. */ + readonly group: string; + /** `@deepseek-ai/cordis-plugin-include`. */ + readonly include: string; /** `@deepseek-ai/cordis-plugin-loader`, what the cluster's range resolves. */ readonly loader: string; /** The committed lockfile that pins this channel, from the repo root. */ @@ -56,33 +65,85 @@ export interface ChannelRecord { readonly piRange: string; /** `@deepseek-ai/schemastery`, what the cluster's `~` range resolves. */ readonly schemastery: string; + /** + * `@deepseek-ai/cordis-plugin-timer`. The only family member this + * repository's own install does not resolve, so no lockfile holds it; the + * `e2e-host` job's floor cell asserts it against the published host's tree. + */ + readonly timer: string; } export const CHANNELS: Readonly> = { alpha: { cordis: "4.0.6-alpha.1", + cosmokit: "1.8.7-alpha.1", distTag: "alpha", dsh: "0.2.2-alpha.2", + group: "1.0.6-alpha.1", + include: "1.0.11-alpha.1", loader: "1.0.7-alpha.1", lockfile: "pnpm-lock.yaml", pi: "1.5.0", piFloor: "1.3.0", piRange: "^1.3.0", schemastery: "3.18.6-alpha.1", + timer: "1.1.8-alpha.1", }, rc: { cordis: "4.0.5", + cosmokit: "1.8.6", distTag: "latest", dsh: "0.2.1-rc.5", + group: "1.0.5", + include: "1.0.10", loader: "1.0.6", lockfile: "channels/rc/pnpm-lock.yaml", pi: "1.5.0", piFloor: "1.1.0", piRange: "^1.1.0", schemastery: "3.18.5", + timer: "1.1.7", }, }; +/** + * The cordis family the published host installs, by package name, at the + * record's versions: cordis, its loader, include, group and timer plugins, + * cosmokit and schemastery. They are released together and peer on one + * another by `~` ranges, so forcing one of them alone (cordis, say) leaves a + * tree nobody ships. + * + * @param record - a channel's record. + * @returns `{ "": "" }` for every family member. + */ +export const cordisFamily = ( + record: ChannelRecord +): Readonly> => ({ + "@deepseek-ai/cordis": record.cordis, + "@deepseek-ai/cordis-plugin-group": record.group, + "@deepseek-ai/cordis-plugin-include": record.include, + "@deepseek-ai/cordis-plugin-loader": record.loader, + "@deepseek-ai/cordis-plugin-timer": record.timer, + "@deepseek-ai/cosmokit": record.cosmokit, + "@deepseek-ai/schemastery": record.schemastery, +}); + +/** + * The npm `overrides` the `e2e-host` job's floor cell installs the published + * host with: the channel's cordis family at the record's versions and pi at + * the bridge range's floor, which is the tree the desktop app ships (its pi + * patched there, not here). The npm cell takes no override. + * + * @param record - a channel's record. + * @returns the overrides, keyed by package name. + */ +export const floorOverrides = ( + record: ChannelRecord +): Readonly> => ({ + ...cordisFamily(record), + "@earendil-works/pi-ai": record.piFloor, +}); + /** The channel the root `package.json` and `pnpm-lock.yaml` install. */ export const DEFAULT_CHANNEL: Channel = "alpha"; diff --git a/test/support/host.ts b/test/support/host.ts index 5f2b145a..7e883d59 100644 --- a/test/support/host.ts +++ b/test/support/host.ts @@ -494,13 +494,17 @@ export const CHARACTERIZED_MODELS: Readonly> = }; /** - * How a characterization row is tagged, so a later phase flips named rows: - * `KNOWN_DEFECT` is wrong today and a phase fixes it, `INTENDED_CHANGE` is - * right today and a phase changes it on purpose, `BASELINE` must survive. - * `HOST_LIMITATION` is something one host cell's pi lacks that the vendor's - * service needs: no phase of this plugin fixes it (D26: a host gap is the - * host's job), its title names the channel and the pi it applies to, and it - * is reported upstream. + * How a characterization row is tagged, so a later phase flips named rows. + * This is the one definition; CONTRIBUTING, the plan and the test files point + * here: + * + * - `KNOWN_DEFECT` is wrong today, and a phase fixes it; + * - `INTENDED_CHANGE` is right today, and a phase changes it on purpose; + * - `BASELINE` must survive every phase; + * - `HOST_LIMITATION` is something one host cell lacks or breaks, its pi or + * its own code, that no phase of this plugin fixes (D26: a host gap is the + * host's job): its title names the channel and the host it applies to, and + * it is reported upstream. */ export type RowTag = | "BASELINE" @@ -846,10 +850,10 @@ const S0_ALPHA_FLOOR = withMoved(S0_RC_FLOOR, { /** * pi 1.5.0 (both channels' newest) files completions under the per-protocol - * family P-R1 targets, and describes both Go gap ids, so each facet that - * moves here is already what P-R1 keeps and is `BASELINE` on this cell: - * completions reach `/inference(/go)/openai/v1`, Go gemini-4-argon validates - * in either apply order, and Go minimax-m3.1-flash is placed (on + * family P-R1 targets, and describes both Go gap ids (`PI_GAPS_BY_CELL`), so + * each facet that moves here is already what P-R1 keeps and is `BASELINE` on + * this cell: completions reach `/inference(/go)/openai/v1`, the Go Gemini row + * validates in either apply order, and the other Go gap id is placed (on * completions, which is the plugin catalog's SDK too) and sent. */ const S0_NEWEST = withMoved( @@ -913,8 +917,8 @@ export const ZEN_UNDESCRIBED = ["gpt-5.3-codex-spark", UNKNOWN_ID] as const; /** * TODAY's rows for an apply order on a host cell. A cell's `rows` is its * `plugin-first` table; with `pi-ai-first` the adapter decoration wraps - * pi-ai's plane adapter and on the floor cells exactly one facet moves — Go - * gemini-4-argon validates (on `newest` pi already places it, so nothing + * pi-ai's plane adapter and on the floor cells exactly one facet moves — the + * Go Gemini row validates (on `newest` pi already places it, so nothing * moves) — * measured in `test/host-characterization.test.ts`, which generates one test * per row and facet for each order, titled with that order's tags. The moved @@ -967,7 +971,7 @@ export const APPLY_ORDERS = Object.keys({ /** * What appending a new id to `opencode` answers while the plane stores ANY of * {@link ZEN_UNDESCRIBED}, whichever one the installed pi cannot describe - * first. Every recorded cell's pi names `gpt-5.3-codex-spark` + * first. Every recorded cell's pi names the first of them * (`S0_TODAY.saveBlockedByStoredGap`); a binary whose npm-resolved pi moved * past its channel record may have learned it and name the next one, so the * binary e2e asserts this instead on such a pi. From c1fca83f081408180b672f6122b6d5a238ea0497 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 00:53:36 +0000 Subject: [PATCH 10/11] fix: hold the cordis family to the declared range floors The record's cordis family meant two things: what the cluster's `~` ranges resolve on npm (the field docs, the drift report, release step 3) and the tree the desktop app ships (the e2e-host floor cell's overrides). They already disagree for alpha. The fields now have one meaning, the floor of the range the published cluster declares, which is what the harness vendors at the tag, what package.json pins and the lockfile holds, and what the floor cell forces. The family moves only with dsh. - fixed: both-kept-2-R2-F1, resolutions-2-T1B-R2-1, rules-docs-2-R2-RD-1. scripts/host-drift.ts compares every family member with the floor of the range its published declarer declares (FAMILY_DECLARERS: the bridge for cordis, the loader and schemastery; dsh for include and timer; dsh-app-boot for group; cordis at the cordis floor for cosmokit), not the newest patch inside it. host-drift.test.ts has a known answer for today's registry (alpha.2 patches inside alpha.2's ranges: no line), the next cut (eight lines, one per moved field) and the timer case. The test failed before the change. Docs follow in host-versions.ts, ci.yml, CONTRIBUTING (release step 3: a family patch under an unchanged dsh is not a bump), D22'' and AGENTS.md. - fixed: both-kept-2-R2-F2, resolutions-2-T1B-R2-2. The record is no longer its own witness. The binary e2e, in both cells, holds each family value to the floor of the range the installed published package declares (a mutated timer fails it). harness-channel.test.ts holds the five members the cluster peers on to the lockfile's declared peer ranges, and the drift report checks the timer plugin nightly. The comments now say what each check asserts. - fixed: both-kept-2-R2-F3, resolutions-2-T1B-R2-3. CONTRIBUTING and the plan make ci.yml's narrower cache claim: a cache filled before the record or its overrides moved is never reused, and the rest of the tree is what npm resolved at the fill. There is no time component in the key. floorOverrides also pins @earendil-works/pi-telemetry at piFloor, so the rc floor cell runs the desktop's pi pair (1.1.0/1.1.0) instead of 1.1.0 with telemetry 1.5.0. - fixed: rules-docs-2-R2-RD-2. Release step 3 and D22'' say a bump that moves piFloor or a family field is green only after a workflow_dispatch run on its branch, which adds the floor cells. The re-measure list names the floor tables (S0_RC_FLOOR/S0_ALPHA_FLOOR and the floor columns of WHOLE_CATALOG and PI_GAPS_BY_CELL), and P-T3's bump-PR row dispatches the floors. The PR matrix stays as decision (3) sets it. - fixed: rules-docs-2-R2-RD-3. RowTag, CONTRIBUTING's Tags paragraph and the plan's tags bullet scope HOST_LIMITATION by channel and install: a host cell for an in-process row, an e2e-host cell (npm or floor) for the binary e2e. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- .github/workflows/ci.yml | 40 +++--- AGENTS.md | 4 +- CONTRIBUTING.md | 8 +- docs/improvement-plan.md | 8 +- scripts/host-drift.ts | 130 ++++++++++++----- test/e2e/host-binary.e2e.ts | 53 ++++++- test/harness-channel.test.ts | 51 ++++++- test/host-drift.test.ts | 260 +++++++++++++++++++++++++--------- test/support/host-versions.ts | 54 ++++--- test/support/host.ts | 10 +- 10 files changed, 455 insertions(+), 163 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70b6c5f3..47aeea56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,12 +107,14 @@ jobs: - name: the switch left the tree clean run: git diff --exit-code && test -z "$(git status --porcelain)" - # The channel records name what each channel installs TODAY, and the harness + # The channel records name each channel's latest release, and the harness # cuts a release without telling this repository. Nightly and on dispatch, - # `scripts/report-host-drift.ts` reads the registry the way npm resolves a - # channel (dsh at its dist-tag; cordis, the loader, schemastery and pi at the - # newest version the bridge's declared ranges admit) and prints one warning - # per field that moved past `test/support/host-versions.ts`. It REPORTS and + # `scripts/report-host-drift.ts` reads the registry the way the records are + # defined (dsh at its dist-tag; pi at the newest version the bridge's range + # admits, and that range's floor; each cordis-family member at the floor of + # the `~` range the tagged cluster declares for it, the desktop's tree, so a + # newer patch inside a range is not drift) and prints one warning per field + # it compares that moved past `test/support/host-versions.ts`. It REPORTS and # never gates, so it exits 0 whatever it finds: a channel moving is the # harness's release, not a regression. P-T3 turns the report into a proposed # bump pull request; the bump follows CONTRIBUTING's release step 3. @@ -404,10 +406,13 @@ jobs: # re-resolving is the point. # - `floor`: the tree the desktop app ships, forced with npm `overrides` # (`floorOverrides` in host-versions.ts): the channel's whole cordis family - # at the record's versions and pi at the bridge range's floor (patched - # there, not here; the patch touches nothing the plugin reads). Cached by - # the channel, its dsh and a hash of those overrides, so a cache filled - # before the record moved is never reused. + # at the record's versions (the floors of the ranges the cluster declares, + # what the harness vendors at the tag) and pi with its telemetry package + # at the bridge range's floor (pi patched there, not here; the patch + # touches nothing the plugin reads). Cached by the channel, its dsh and a + # hash of those overrides, so a cache filled before the record or its + # overrides moved is never reused; the rest of the tree is what npm + # resolved when the cache was filled. # Pull requests run the `npm` cell of both channels, and only when they touch # the host seam; dispatch and the nightly run add both `floor` cells. # @@ -488,8 +493,8 @@ jobs: node --experimental-strip-types --input-type=module -e "import { createHash } from 'node:crypto'; import { CHANNELS, floorOverrides } from './test/support/host-versions.ts'; const r = CHANNELS[process.env.DSH_CHANNEL]; const overrides = JSON.stringify(floorOverrides(r)); console.log('dsh=' + r.dsh); console.log('pi=' + r.pi); console.log('piRange=' + r.piRange); console.log('floorOverrides=' + overrides); console.log('floorKey=' + createHash('sha256').update(overrides).digest('hex').slice(0, 16))" >> "$GITHUB_OUTPUT" echo "PREFIX=$RUNNER_TEMP/dsh-host" >> "$GITHUB_ENV" # The floor install forces every version that matters (the record's - # cordis family and the floor pi), so it is cached by the channel, its - # dsh and a hash of those overrides; the npm one is the point of + # cordis family and the floor pi pair), so it is cached by the channel, + # its dsh and a hash of those overrides; the npm one is the point of # re-resolving, so never. - name: cache the floor host install id: host-cache @@ -532,11 +537,14 @@ jobs: if [ "$CELL" = npm ] && [ "$resolved" != "$PI" ]; then echo "::warning::npm resolved pi $resolved for ${DSH_CHANNEL}, not the record's $PI: the rows that depend on which ids pi describes are skipped" fi - # Every package the floor cell forces (pi and the cordis family): the - # floor cell must hold exactly the record's version, a cached prefix - # included; the npm cell reports what npm resolved, which is the - # channel's release as a user gets it (the nightly drift report - # names a move past the record). + # Every package the floor cell forces (the pi pair and the cordis + # family): the floor cell must hold exactly the override, a cached + # prefix included, which catches npm ignoring one; the npm cell + # reports what npm resolved, which is the channel's release as a + # user gets it. Whether the record's values are right is not this + # step's: the binary e2e holds them to the ranges the installed + # published packages declare, and the nightly drift report to the + # registry's. failed=0 for name in $(node -p "Object.keys(JSON.parse(process.env.FLOOR_OVERRIDES)).join(' ')"); do want="$(node -p "JSON.parse(process.env.FLOOR_OVERRIDES)['$name']")" diff --git a/AGENTS.md b/AGENTS.md index ce492ebf..a7d96723 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,11 +42,11 @@ Host bundle (`lib/index.mjs`), a thin `apply` barrel over small modules: - `src/index.ts` · `src/identity.ts` · `src/lifecycle.ts` — the public barrel, the identity constants, and the `apply()` composition (one installer per side effect, in load-bearing order). The barrel is the host's contract and nothing more: `apply`, `Config`, `inject`, `name` and the config types. Everything else is internal, and `test/public-api.test.ts` pins the list, so widening it is a reviewed change. - `test/support/src.ts` — the one door tests use into `src`: each name re-exported from the module that DEFINES it, never through the barrel, so a module move edits one file. `test/architecture.test.ts` enforces both halves — every name is declared in its target, and no test outside the named exemptions (`vi.mock` targets, fresh re-imports after `vi.resetModules()`, the architecture and public-API tests, the jsdom client tests, the `reset-process-state.ts` setup file) imports a `src` path, and no jsdom test loads the door. -- `test/support/host-versions.ts` — the hosts the plugin supports, named once: one record per harness channel (`rc` = `dsh@latest`, `alpha` = `dsh@alpha`), each with its `@deepseek-ai` cluster, cordis, its loader, the bridge's pi RANGE, the newest pi that range resolves (the channel lockfile's) and the range's floor (the harness's own pin, which only the desktop app ships), plus which one the install under test is (`DSH_CHANNEL`, else the marker `scripts/use-channel.ts` leaves, else `alpha`). `test/host-versions.test.ts` checks the installed tree against the SELECTED record; `test/harness-channel.test.ts` checks the declarations and holds each committed lockfile (`pnpm-lock.yaml` for alpha, `channels/rc/pnpm-lock.yaml` for rc) to its record, including that `@deepseek-ai/schemastery` resolves to one copy inside the cluster's range. +- `test/support/host-versions.ts` — the hosts the plugin supports, named once: one record per harness channel (`rc` = `dsh@latest`, `alpha` = `dsh@alpha`), each with its `@deepseek-ai` cluster, the cordis family (`cordisFamily`: cordis, the loader, include, group, timer, cosmokit, schemastery, each at the floor of the `~` range the published cluster declares, which is what the harness vendors at the tag and moves only with `dsh`), the bridge's pi RANGE, the newest pi that range resolves (the channel lockfile's) and the range's floor (the harness's own pin, which only the desktop app ships), and `floorOverrides`, the family and the floor pi pair the `e2e-host` floor cell forces, plus which one the install under test is (`DSH_CHANNEL`, else the marker `scripts/use-channel.ts` leaves, else `alpha`). `test/host-versions.test.ts` checks the installed tree against the SELECTED record; `test/harness-channel.test.ts` checks the declarations and holds each committed lockfile (`pnpm-lock.yaml` for alpha, `channels/rc/pnpm-lock.yaml` for rc) to its record, including that `@deepseek-ai/schemastery` resolves to one copy inside the cluster's range and that each family member the cluster peers on is the floor of the declared range. - `test/support/host.ts` — the real host, in-process, in two tiers: **A** (`mountTierA`: a bare cordis `Context`, `LlmRuntime` and `dsh-llm-pi-ai` mounted with `{ providers }`) for the `internal/config` waterfall and pi-ai's credential paths, and **B** (`bootTierB`: dsh-app-boot's `boot()` with a temporary HOME and a test bundle) for the config editor, `settings.describe`, the volatile commit and the characterization of the plugin as it ships. Its fakes (`fakeCredentials`, `fakeLaunchEnvironment`, `recordingFetch`) keep every case offline and its credentials out of the environment (a boot blanks the two keys the plugin reads directly, should the shell export them, and restores them on dispose); `S0_PROFILE` is the contract fixture, `CHARACTERIZED_PROFILE` (S0 plus a declared catalog-unknown id) the characterization's, and `S0_TODAY_BY_CELL` the measured per-model rows, tagged per facet, one table per host cell (`newest`, either channel's default install, and `rc-floor`/`alpha-floor`, the pis the desktop app ships; `S0_TODAY` is the installed cell's, and `forHost`/`forChannel` pick any other per-cell expectation), that the binary e2e shares (`todayRowsFor` picks them by the cell and by the apply order the probe infers from whether the plugin's decoration wrapped pi-ai's plane adapter). Tier A has no loader, so this plugin mounted there cannot find pi-ai and mounts no hidden route and no Google bridge (whose adapter class comes from the same loader entry, `loadPiAiAdapterHost`); `test/host-harness.test.ts` exercises the builder options no contract or characterization case passes, and tier B's two failure paths (an invalid first row, an exported key). Tier B's rows carry PACKAGE names through a stub module loader, never `cordis:` builtins, because the plugin finds pi-ai's entry by `options.name` (`piAiEntry` in `src/responses-provider.ts`) and a builtin-named row would drop the hidden-route mount from the characterization. It also fixes which of pi-ai and the plugin applies first (`order`), because the loader starts rows in parallel and the published binary applies the plugin first; the held row is released when the first one applies or its fiber settles, so an invalid first row fails instead of hanging the boot. - `test/support/keyless-matrix.ts` — the keyless 401-vs-404 probe matrix's cell table and classifier. Pure, though it lives in `test/support/` rather than `src/utils/`: a function of its arguments, with no code path that can set a credential header; `test/e2e/keyless-matrix.e2e.ts` sends the cells. - `test/e2e/host/` — the measurement bundle (`probe-plugin.mjs`, its manifest and patch) the `e2e-host` job links into a throwaway profile under a temporary HOME, through a relative `link:` path. `test/e2e/host-binary.e2e.ts` is its reader, gated on `DSH_HOST_E2E=1`; it installs no host itself and needs `DSH_HOST_PREFIX`, a prefix holding the installed one. -- `scripts/report-host-drift.ts` — the nightly harness-drift report (CI's `host-drift` job, schedule and dispatch only): reads the registry the way npm resolves each channel and prints one `::warning::` per record field that moved, exiting 0 whatever it finds, because a channel moving is the harness's release, not a regression. Its rules are `scripts/host-drift.ts` (`hostDrift`, pure), held by `test/host-drift.test.ts` with known answers, a loopback registry and the job's evaluated condition. P-T3 turns the report into a proposed bump pull request. +- `scripts/report-host-drift.ts` — the nightly harness-drift report (CI's `host-drift` job, schedule and dispatch only): reads the registry the way the records are defined (`dsh` at its dist-tag, pi's range, floor and newest, and every cordis-family member at the floor of the range the tagged cluster declares) and prints one `::warning::` per compared field that moved, exiting 0 whatever it finds, because a channel moving is the harness's release, not a regression. Its rules are `scripts/host-drift.ts` (`hostDrift`, pure), held by `test/host-drift.test.ts` with known answers, a loopback registry and the job's evaluated condition. P-T3 turns the report into a proposed bump pull request. - `scripts/use-channel.ts` — switches the dev install: `rc` or `alpha` installs that channel's committed lockfile (unpatched pi at the newest version the bridge's range admits, what an npm user runs), and `--floor` the same cluster with pi at the range's floor, never committed. It restores every tracked file it touched, so a switch leaves no diff. Its pure half is `scripts/channel-manifest.ts` (`channelManifest`, `pinFor`, `workspaceWithPi`, and `withoutHostPackages`, the lockfile `rc --relock` starts from), held by `test/use-channel.test.ts`. `rc --relock` belongs in the same commit as ANY change to `package.json`'s dependencies or `pnpm-workspace.yaml`'s catalog or overrides, not only a channel bump: the rc lockfile carries every specifier the root one does and CI installs it frozen, and `test/harness-channel.test.ts` fails, naming the command, when they disagree. Dependabot ignores the cluster and pi, which move only as a channel bump. **This repository carries no pi patch** (D14''): an npm install runs pi unpatched, the harness's patch reaches users only through the desktop app and touches nothing the plugin reads, and a version-keyed patch would break every pi bump. `test/architecture.test.ts` keeps it that way by failing if `src` ever reads a streamed tool-call delta, partial arguments or a thought signature. - `src/config.ts` · `src/config-values.ts` · `src/utils/guards.ts` — the schema and its defaults; dependency-free readers and type guards. The client bundle may import these, and it does — along with the contract and presentation modules (`usage-contract.ts`, `settings-copy.ts`, `settings-fields.ts`, `usage-ui.ts`, `go-limits.ts`, `opencode-store-contract.ts`). The rule is not a list of names: **the client may import a module that reaches nothing host-only** — no cordis context, no credential seam, no node built-in — and the bundle gate is what enforces it, by failing the build rather than by anyone remembering. An earlier version of this line named two files as the only ones allowed, which the tree had already outgrown. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbd498eb..45f9be41 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ What every case does: - **Give tier-B cases 30 s**, and wait with a deadline (`waitForRoutes`, 10 s by default, so a route that never registers fails by name inside the hook's budget), never a fixed sleep: the plugin mounts its hidden routes asynchronously. - **A first row that fails does not hang the boot.** Tier B holds the second of pi-ai and the plugin until the first applies OR its fiber settles in any state, so a row whose config the schema rejects boots with that fiber FAILED (`fiber.await()` rethrows the real error) and the other row applied. -**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one cell's host lacks or breaks, its pi or its own code, that no phase of this plugin fixes: the title names the channel and the host, and it is reported upstream, D26). `RowTag` in `test/support/host.ts` holds the one definition of the four. A facet's tag is per cell: one that pi 1.5.0 already serves the way P-R1 keeps (completions on the per-protocol family, Go `gemini-4-argon` validating, Go `minimax-m3.1-flash` placed) is `[BASELINE]` on `newest` and keeps its floor tag on the floors. A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon`'s call config, on the floors only: pi 1.5.0 places the id in either order) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. +**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one channel's host, as one install runs it, lacks or breaks, its pi or its own code, that no phase of this plugin fixes: the install is a host cell for an in-process row, or an `e2e-host` cell, `npm` or `floor`, for the binary e2e; the title names the channel, the host and that install, and it is reported upstream, D26). `RowTag` in `test/support/host.ts` holds the one definition of the four. A facet's tag is per cell: one that pi 1.5.0 already serves the way P-R1 keeps (completions on the per-protocol family, Go `gemini-4-argon` validating, Go `minimax-m3.1-flash` placed) is `[BASELINE]` on `newest` and keeps its floor tag on the floors. A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor(order)` carries the pi-ai-first table's own tags, where Go Gemini's call config is already the behaviour P-R1 keeps), so the tags in `S0_TODAY` are what govern which entries a phase may flip. The one facet that depends on the apply order today (Go `gemini-4-argon`'s call config, on the floors only: pi 1.5.0 places the id in either order) is pinned by its own `[KNOWN_DEFECT]` case, outside the booted blocks; the phase that flips that row also drops `todayRowsFor`'s pi-ai-first override and turns that case into a `[BASELINE]` "no facet depends on the apply order" check, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. **A catalog refresh can touch the rows.** `S0_TODAY`'s context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case compares each row's context window and effort ids to the bundled catalog's derivation and names the model that moved; when it fails on a refresh pull request, update the row in `S0_RC_FLOOR` (the context windows and efforts every cell inherits), and in a cell's `withMoved` entry where that cell restates the row. A phase that moves a non-`BASELINE` `info` facet off the catalog on purpose (the user's context window, a toggle model's Off/On) adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. @@ -79,7 +79,7 @@ What every case does: `.github/workflows/ci.yml`'s `e2e-host` job installs each channel's published `@deepseek-ai/dsh` (its record's `dsh`) into a throwaway prefix with npm, the way a user installs it, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the temporary home's credentials document (`$DSH_HOME/.credentials.yaml`, shared by every profile under that home), boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. -- **Cells, per channel, pi unpatched in both.** `npm` is the published host exactly as npm resolves it, cordis and pi included, what an npm user runs, and asserts the `newest` rows while its pi is the channel record's; a pi published since the record still has to sit in the bridge's range, but the rows that depend on which ids pi describes (the keyed listing and the per-model rows) are skipped with a warning, and the nightly `host-drift` job names the new pi. `floor` is the tree the desktop app ships: npm `overrides` (`floorOverrides` in `host-versions.ts`) force the channel's whole cordis family (cordis, the loader, the include, group and timer plugins, cosmokit, schemastery) at the record's versions and pi at the range's floor; the job and the e2e assert each one, and the cell asserts every row of the channel's floor cell. It is cached by channel, `dsh` and a hash of those overrides, so it never runs a tree frozen when the cache was filled. The installed `dsh` must be the channel record's, so a prefix of the wrong channel fails by name. The save rows run whatever pi resolved (they do not depend on which ids pi describes, though they assume pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage that as a pi change first); they skip only where the host itself cannot save (next bullet). +- **Cells, per channel, pi unpatched in both.** `npm` is the published host exactly as npm resolves it, cordis and pi included, what an npm user runs, and asserts the `newest` rows while its pi is the channel record's; a pi published since the record still has to sit in the bridge's range, but the rows that depend on which ids pi describes (the keyed listing and the per-model rows) are skipped with a warning, and the nightly `host-drift` job names the new pi. `floor` is the tree the desktop app ships: npm `overrides` (`floorOverrides` in `host-versions.ts`) force the channel's whole cordis family (cordis, the loader, the include, group and timer plugins, cosmokit, schemastery) at the record's versions, which are the floors of the `~` ranges the published cluster declares (what the harness vendors at the tag), and pi with its telemetry package at the range's floor; the job and the e2e assert that npm applied each one, and the cell asserts every row of the channel's floor cell. In both cells the e2e also holds each record value to the floor of the range the installed published package declares for it, which no override rewrites. It is cached by channel, `dsh` and a hash of those overrides, so a cache filled before the record or its overrides moved is never reused; the rest of the tree is what npm resolved when the cache was filled. The installed `dsh` must be the channel record's, so a prefix of the wrong channel fails by name. The save rows run whatever pi resolved (they do not depend on which ids pi describes, though they assume pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage that as a pi change first); they skip only where the host itself cannot save (next bullet). - **The host's own failures are recorded, not worked around.** Only the `npm` cell lets cordis float, as the published host's `~` ranges do for a user, and the job reports what it resolved. Where that leaves the host unable to run its own save path, the run records a `[HOST_LIMITATION]` row (titled with the channel and the `dsh`) and skips the save rows. It is detected from what the host answered, never from a version (D26): the plane edit and a control edit no plugin's listener reads (the probe's own entry) both fail with the same `TypeError` thrown by `@deepseek-ai/dsh-app-boot`; where the host saves, the row skips and a `[BASELINE]` row asserts the control was accepted. Today that is alpha's `0.2.2-alpha.2` as npm installs it: cordis `4.0.6-alpha.2` calls `internal/update` in a new shape while `dsh-app-boot` registers its listener in the old one, so every config-editor save throws `next is not a function`. The alpha floor, on the cordis the harness vendors at the tag, saves. - **Triggers.** Dispatch and the nightly run take both cells of both channels. A pull request takes the `npm` cell of both channels, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` (what the job installs), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. @@ -95,7 +95,7 @@ pnpm run build DSH_HOST_E2E=1 DSH_HOST_PREFIX=/home/you/dsh-host pnpm run test:e2e:host --run ``` -For the `floor` cell, use a fresh prefix whose `package.json` carries the channel's `floorOverrides` (the cordis family at the record's versions and pi at the range's floor) as `"overrides"` before the install, written the same way the CI job writes it, and run the suite with `DSH_HOST_PI_CELL=floor`: +For the `floor` cell, use a fresh prefix whose `package.json` carries the channel's `floorOverrides` (the cordis family at the record's versions and pi with its telemetry package at the range's floor) as `"overrides"` before the install, written the same way the CI job writes it, and run the suite with `DSH_HOST_PI_CELL=floor`: ```sh mkdir -p /home/you/dsh-host-floor @@ -161,7 +161,7 @@ git diff --name-only main... \ 1. Use conventional commits (`feat:`, `fix:`) — release-please opens the version-bump + changelog PR automatically; merging it tags and creates the GitHub Release, which fires OIDC publishing to npm and GitHub Packages. 2. Avoid manual releases: release-please anchors on the latest GitHub **Release**, not the latest tag, so a bare tag is invisible to it (`AGENTS.md`, "Releasing"). If one is unavoidable, bump `version` in `package.json`, commit, tag, push the tag, and create the GitHub Release in the same step (`gh release create vX.Y.Z --generate-notes`). Never touch `CHANGELOG.md`: release-please owns it. -3. On every release, after smoke-testing (registry install resolves, plugin loads in the DSH Web profile, and a free-tier Zen call succeeds), check the **Supported: the latest release of both DeepSeek Harness channels** line and the **Support policy** paragraph under "Compatibility & Verification" in both READMEs. Their "today" versions, pi ranges and floors, and the default install named a paragraph below, are copied from the channel records in `test/support/host-versions.ts`, and `test/harness-channel.test.ts` holds them to the records. A channel bump **inside the prerelease tuples the peer ranges already admit** (a loader or schemastery patch that changes no host hook, say) moves version data, and this is its whole set: for `rc`, the rc record and `channels/rc/pnpm-lock.yaml` (`scripts/use-channel.ts rc --relock`); for `alpha`, the alpha record, `package.json`'s exact `@deepseek-ai/*` pins and the root lockfile (`pnpm install`); for either, those README sentences in both languages, and `minimumReleaseAgeExclude` lines in `pnpm-workspace.yaml` for the exact versions when the cut is younger than pnpm's one-day release-age window. A pi move is the same data: `pnpm install` and `--relock` keep the pi a lockfile already satisfies, so move it with `pnpm update @earendil-works/pi-ai --no-save` (the root lockfile; `package.json` keeps the union range), then `scripts/use-channel.ts rc --relock`, which carries the root's pi into the rc lockfile, and set each record's `pi`. The same update is what leaves one pi when a new bridge raises its floor above the locked one. The F10 characterization (`test/mount-protocols.test.ts`) is keyed by pi line, so a pi inside a line leaves it alone; a pi outside every line is a new major, and a reviewed change. **Data only while the host behaves the same.** A bump stays version data while tiers A and B and the binary e2e stay green on the new install. One that turns them red because the host behaves differently is a reviewed change made in the bump's own pull request and named in its body: a pi whose catalog or bases moved (a model added or dropped, a gap id learned) re-measures the `newest` cell's tables (`WHOLE_CATALOG` in `test/host-shapes.test.ts`, `PI_GAPS_BY_CELL` and `S0_NEWEST` in `test/support/host.ts`), and a host event whose call shape or waterfall order moved adapts the tier listeners in `test/support/host.ts` and the contract tests, re-measuring the facts they pin (F8, F12, F14, F20) rather than accepting both shapes. The next alpha cut is that case: cordis `4.0.6-alpha.2` changed the call shape of `internal/config` and `internal/update`, so the `0.2.2-alpha.3` bump rewrites the tier listeners (a follow-up to the restack). A cut that **opens a new prerelease tuple** (dsh `0.2.3-*`, cordis `4.0.7-*`) is not data: semver admits a prerelease only through a comparator in its own tuple, so it is a reviewed range change that also adds the matching `>=X.Y.Z-0 =X.Y.Z-0 > { readonly distTag: string; readonly dsh: string; - readonly loader: string; readonly pi: string; readonly piFloor: string; readonly piRange: string; - readonly schemastery: string; } /** The slice of an npm packument (abbreviated or full) the comparison reads. */ @@ -50,12 +61,11 @@ export interface Packument { /** The packages the CLI fetches, by the role each plays in a record. */ export const REGISTRY_PACKAGES = { + appBoot: "@deepseek-ai/dsh-app-boot", bridge: "@deepseek-ai/dsh-llm-pi-ai", cordis: "@deepseek-ai/cordis", dsh: "@deepseek-ai/dsh", - loader: "@deepseek-ai/cordis-plugin-loader", pi: "@earendil-works/pi-ai", - schemastery: "@deepseek-ai/schemastery", } as const; export type RegistryRole = keyof typeof REGISTRY_PACKAGES; @@ -63,13 +73,52 @@ export type RegistryRole = keyof typeof REGISTRY_PACKAGES; /** One packument per role: what the registry answered. */ export type RegistryView = Readonly>; -/** The roles whose version follows a range the bridge declares. */ -const RESOLVED: readonly ("cordis" | "loader" | "schemastery")[] = [ - "cordis", - "loader", - "schemastery", +/** + * Which published package declares each family member's range. The cluster's + * packages are read at the record's `dsh`; cosmokit is declared by cordis, + * read at the cordis floor. Cordis comes first, so its floor is known by the + * time cosmokit is read. The binary e2e reads the same table off the + * installed host. + */ +export const FAMILY_DECLARERS: readonly { + readonly declarer: "appBoot" | "bridge" | "cordis" | "dsh"; + readonly field: FamilyField; + readonly name: string; +}[] = [ + { declarer: "bridge", field: "cordis", name: "@deepseek-ai/cordis" }, + { + declarer: "bridge", + field: "loader", + name: "@deepseek-ai/cordis-plugin-loader", + }, + { + declarer: "bridge", + field: "schemastery", + name: "@deepseek-ai/schemastery", + }, + { + declarer: "dsh", + field: "include", + name: "@deepseek-ai/cordis-plugin-include", + }, + { declarer: "dsh", field: "timer", name: "@deepseek-ai/cordis-plugin-timer" }, + { + declarer: "appBoot", + field: "group", + name: "@deepseek-ai/cordis-plugin-group", + }, + { declarer: "cordis", field: "cosmokit", name: "@deepseek-ai/cosmokit" }, ]; +/** A manifest's declared ranges, peers and dependencies together. */ +export const declaredRanges = (manifest: { + readonly dependencies?: Readonly>; + readonly peerDependencies?: Readonly>; +}): Readonly> => ({ + ...manifest.peerDependencies, + ...manifest.dependencies, +}); + /** The newest published version `range` admits, as npm resolves it. */ export const newestIn = ( packument: Packument, @@ -88,8 +137,8 @@ export const floorOf = (range: string): string | undefined => * One line per field where the registry and a channel's record disagree, each * naming the channel, the field, the record's value and the registry's. An * empty list means every record is current. A registry answer that cannot be - * read (no dist-tag, no bridge at that version, no declared range, nothing in - * the range) is a line too: "read nothing" must never look like "current". + * read (no dist-tag, no declaring package at that version, no declared range, + * nothing in the pi range) is a line too: "read nothing" must never look like "current". */ export const hostDrift = ( records: Readonly>, @@ -119,10 +168,7 @@ export const hostDrift = ( ); continue; } - const declared = { - ...bridge.peerDependencies, - ...bridge.dependencies, - } as Readonly>; + const declared = declaredRanges(bridge); const piRange = declared[REGISTRY_PACKAGES.pi]; if (piRange === undefined) { @@ -147,20 +193,38 @@ export const hostDrift = ( } } - for (const role of RESOLVED) { - const name = REGISTRY_PACKAGES[role]; - const range = declared[name]; + const floors: Partial> = {}; + for (const { declarer, field, name } of FAMILY_DECLARERS) { + const at = declarer === "cordis" ? floors.cordis : tagged; + const declarerName = REGISTRY_PACKAGES[declarer]; + if (at === undefined) { + lines.push( + `${channel}: no ${declarerName} floor was read; ${name} was not compared` + ); + continue; + } + const manifest = registry[declarer].versions?.[at]; + if (manifest === undefined) { + lines.push( + `${channel}: ${declarerName}@${at} is not published; ${name} was not compared` + ); + continue; + } + const range = declaredRanges(manifest)[name]; if (range === undefined) { + lines.push(`${channel}: ${declarerName}@${at} declares no ${name}`); + continue; + } + const floor = floorOf(range); + if (floor === undefined) { lines.push( - `${channel}: ${REGISTRY_PACKAGES.bridge}@${tagged} declares no ${name}` + `${channel}: ${declarerName}@${at} declares ${name} ${range}, which has no floor` ); continue; } - const found = newestIn(registry[role], range); - if (found === undefined) { - lines.push(`${channel}: no published ${name} satisfies ${range}`); - } else if (found !== record[role]) { - say(`${name} (${range})`, record[role], found); + floors[field] = floor; + if (floor !== record[field]) { + say(`the floor of ${name} ${range}`, record[field], floor); } } } diff --git a/test/e2e/host-binary.e2e.ts b/test/e2e/host-binary.e2e.ts index 7d983730..5611315f 100644 --- a/test/e2e/host-binary.e2e.ts +++ b/test/e2e/host-binary.e2e.ts @@ -37,9 +37,15 @@ * least one of `ZEN_UNDESCRIBED` and still describes every other declared * Zen id, so a pi that learns all of them or retires a declared id turns * them red by design — triage that as a pi change first. - * - `floor`: the tree the desktop app ships: the channel's cordis family at - * the record's versions (`cordisFamily`) and pi at the range's floor, both - * asserted here. Every row is the channel's floor cell. + * - `floor`: the tree the desktop app ships: npm `overrides` + * (`floorOverrides`) force the channel's cordis family at the record's + * versions and pi (with its telemetry package) at the range's floor; this + * file asserts npm applied each one. Every row is the channel's floor cell. + * + * In both cells the record's family is held to something the record does not + * produce: each member must be the floor of the `~` range the installed + * published package declares for it (`FAMILY_DECLARERS`; overrides rewrite + * what is installed, never a manifest's declared range). * * Only the `npm` cell lets cordis float, as the published host's `~` ranges * do for a user. Where what it resolved leaves the host unable to run its own @@ -74,9 +80,15 @@ import { fileURLToPath } from "node:url"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + declaredRanges, + FAMILY_DECLARERS, + floorOf, + REGISTRY_PACKAGES, +} from "../../scripts/host-drift.ts"; import { CHANNELS, - cordisFamily, + floorOverrides, HOST_VERSIONS, } from "../support/host-versions.ts"; import { @@ -362,15 +374,42 @@ describe.skipIf(!ENABLED)("e2e · the published host binary (dsh)", () => { ) ) as { version: string } ).version; + const installedManifest = (name: string) => + JSON.parse( + readFileSync( + join(PREFIX, "node_modules", ...name.split("/"), "package.json"), + "utf8" + ) + ) as { + dependencies?: Record; + peerDependencies?: Record; + version: string; + }; if (CELL === "floor") { expect(piVersion).toBe(RECORD.piFloor); - // The floor cell is the tree the desktop app ships, so the cordis family - // is the record's, not whatever the host's `~` ranges resolve today. - for (const [name, version] of Object.entries(cordisFamily(RECORD))) { + // npm applied every override: the floor cell is the tree the desktop + // app ships, not whatever the host's `~` ranges resolve today. + for (const [name, version] of Object.entries(floorOverrides(RECORD))) { expect(installedVersion(name), name).toBe(version); } } cordisVersion = installedVersion("@deepseek-ai/cordis"); + // The record's family is the floor of the range the installed published + // package declares (cosmokit's declarer, cordis, only where the record's + // cordis is the one installed): an independent witness in either cell. + const witnessed: string[] = []; + for (const { declarer, field, name } of FAMILY_DECLARERS) { + const manifest = installedManifest(REGISTRY_PACKAGES[declarer]); + if (declarer === "cordis" && manifest.version !== RECORD.cordis) { + continue; + } + expect( + floorOf(declaredRanges(manifest)[name] ?? ""), + `${REGISTRY_PACKAGES[declarer]}@${manifest.version} declares ${name}` + ).toBe(RECORD[field]); + witnessed.push(field); + } + expect(witnessed).toContain("timer"); rowsHold = piVersion === ROWS_PI; if (!rowsHold) { console.warn( diff --git a/test/harness-channel.test.ts b/test/harness-channel.test.ts index c306cbc6..5c543d93 100644 --- a/test/harness-channel.test.ts +++ b/test/harness-channel.test.ts @@ -41,6 +41,7 @@ import { describe, expect, it } from "vitest"; import { parse, parseAllDocuments } from "yaml"; import { channelManifest, pinFor } from "../scripts/channel-manifest.ts"; +import { floorOf } from "../scripts/host-drift.ts"; import { type Channel, CHANNELS, @@ -473,8 +474,8 @@ describe.each(Object.keys(CHANNELS) as Channel[])( // `cordisFamily` is what the `e2e-host` job forces on the floor cell, so // a family member here and the record disagreeing would make that cell // run a tree this install never measured. The timer plugin is the one - // member nothing here depends on: no lockfile resolves it, and the floor - // cell asserts it against the published host's tree instead. + // member nothing here depends on, so no lockfile resolves it; the next + // case and the drift report hold it instead. const lock = lockfile(record.lockfile); const family = Object.entries(cordisFamily(record)); expect(family).toHaveLength(7); @@ -487,9 +488,55 @@ describe.each(Object.keys(CHANNELS) as Channel[])( expect(floorOverrides(record)).toStrictEqual({ ...Object.fromEntries(family), "@earendil-works/pi-ai": record.piFloor, + "@earendil-works/pi-telemetry": record.piFloor, }); }); + it("names each family member at the floor of the range the published cluster declares", () => { + // The record is not its own witness: a family version must be the floor + // of every `~` range the channel's published packages declare for it. + // A lockfile keeps a package's declared PEER ranges (its dependencies + // only as resolved versions), so this reads every cluster package's + // peers. That floor is what the harness vendors at the tag and the + // desktop app ships; a newer patch inside the range is the npm cell's, + // never the record's. cosmokit (a dependency of cordis) and the timer + // plugin (declared by packages no lockfile here holds) are not peers + // here; the drift report and the binary e2e read them off the manifests. + const { packages } = parse(lockfile(record.lockfile).project) as { + packages: Record }>; + }; + const family = cordisFamily(record); + const checked = new Set(); + for (const [key, entry] of Object.entries(packages)) { + if ( + !key.startsWith("@deepseek-ai/") || + !(key.endsWith(`@${record.dsh}`) || key.endsWith(`@${record.cordis}`)) + ) { + continue; + } + for (const [name, range] of Object.entries( + entry.peerDependencies ?? {} + )) { + const version = family[name]; + if (version === undefined) { + continue; + } + expect(floorOf(range), `${key} declares ${name} ${range}`).toBe( + version + ); + checked.add(name); + } + } + // Known answer: the five members the cluster peers on. + expect([...checked].toSorted()).toStrictEqual([ + "@deepseek-ai/cordis", + "@deepseek-ai/cordis-plugin-group", + "@deepseek-ai/cordis-plugin-include", + "@deepseek-ai/cordis-plugin-loader", + "@deepseek-ai/schemastery", + ]); + }); + it("holds no harness package on the other channel's prerelease line", () => { // The checks above name dsh, cordis, the loader and schemastery; a // relock that kept the other channel's optional peers (`-include`, diff --git a/test/host-drift.test.ts b/test/host-drift.test.ts index d3eb4092..9ff9ac7f 100644 --- a/test/host-drift.test.ts +++ b/test/host-drift.test.ts @@ -47,80 +47,128 @@ const versions = (...list: string[]): Packument => ({ versions: Object.fromEntries(list.map((version) => [version, {}])), }); +/** + * The ranges a channel's published cluster declares for the cordis family: + * each member's `~` range opens at the record's version, as the harness's own + * workspace versions do at the tag. + */ +const declaredBy = ( + record: DriftRecord +): Record<"appBoot" | "bridge" | "cordis" | "dsh", Packument["versions"]> => ({ + appBoot: { + [record.dsh]: { + peerDependencies: { + "@deepseek-ai/cordis-plugin-group": `~${record.group}`, + }, + }, + }, + bridge: { + [record.dsh]: { + dependencies: { + [REGISTRY_PACKAGES.pi]: record.piRange, + "@deepseek-ai/schemastery": `~${record.schemastery}`, + }, + peerDependencies: { + [REGISTRY_PACKAGES.cordis]: `~${record.cordis}`, + "@deepseek-ai/cordis-plugin-loader": `~${record.loader}`, + }, + }, + }, + cordis: { + [record.cordis]: { + dependencies: { "@deepseek-ai/cosmokit": `~${record.cosmokit}` }, + }, + }, + dsh: { + [record.dsh]: { + dependencies: { + "@deepseek-ai/cordis-plugin-include": `~${record.include}`, + "@deepseek-ai/cordis-plugin-timer": `~${record.timer}`, + }, + }, + }, +}); + /** * The registry exactly as the records describe it: each channel's dsh under - * its dist-tag, a bridge at that version declaring the record's ranges, and - * every recorded version published. + * its dist-tag, its cluster at that version declaring the record's ranges, + * cordis at the record's version declaring cosmokit's, and every recorded pi + * published. */ const registryOf = ( records: Readonly> ): RegistryView => { const all = Object.values(records); + const merged = (role: keyof ReturnType): Packument => ({ + versions: Object.assign( + {}, + ...all.map((record) => declaredBy(record)[role]) + ), + }); return { - bridge: { - versions: Object.fromEntries( - all.map((record) => [ - record.dsh, - { - dependencies: { - [REGISTRY_PACKAGES.pi]: record.piRange, - [REGISTRY_PACKAGES.schemastery]: `~${record.schemastery}`, - }, - peerDependencies: { - [REGISTRY_PACKAGES.cordis]: `~${record.cordis}`, - [REGISTRY_PACKAGES.loader]: `~${record.loader}`, - }, - }, - ]) - ), - }, - cordis: versions(...all.map((record) => record.cordis)), + appBoot: merged("appBoot"), + bridge: merged("bridge"), + cordis: merged("cordis"), dsh: { "dist-tags": Object.fromEntries( all.map((record) => [record.distTag, record.dsh]) ), + ...merged("dsh"), }, - loader: versions(...all.map((record) => record.loader)), pi: versions( ...all.flatMap((record) => [record.pi, record.piFloor]), // Never the newest 1.x: a prerelease and the next major. "1.99.0-beta.1", "2.0.0" ), - schemastery: versions(...all.map((record) => record.schemastery)), }; }; -/** Today's alpha cut: dsh moved, the cluster's ranges with it. */ -const ALPHA_3: RegistryView = (() => { +/** Every family member one patch newer than the alpha record, as alpha.2. */ +const ALPHA_PATCHED: DriftRecord = { + ...CHANNELS.alpha, + cordis: "4.0.6-alpha.2", + cosmokit: "1.8.7-alpha.2", + group: "1.0.6-alpha.2", + include: "1.0.11-alpha.2", + loader: "1.0.7-alpha.2", + schemastery: "3.18.6-alpha.2", + timer: "1.1.8-alpha.2", +}; + +/** + * Today's registry, measured 2026-11-03: every family member has an + * `-alpha.2` patch inside the `~` ranges the alpha.2 cluster declares, and + * cordis `4.0.6-alpha.2` declares the patched cosmokit. The cluster itself + * still declares the `-alpha.1` floors. + */ +const TODAY: RegistryView = (() => { const base = registryOf(CHANNELS); return { ...base, - bridge: { + cordis: { versions: { - ...base.bridge.versions, - "0.2.2-alpha.3": { - dependencies: { - [REGISTRY_PACKAGES.pi]: CHANNELS.alpha.piRange, - [REGISTRY_PACKAGES.schemastery]: "~3.18.6-alpha.2", - }, - peerDependencies: { - [REGISTRY_PACKAGES.cordis]: "~4.0.6-alpha.2", - [REGISTRY_PACKAGES.loader]: "~1.0.7-alpha.2", - }, - }, + ...base.cordis.versions, + ...declaredBy(ALPHA_PATCHED).cordis, }, }, - cordis: versions(CHANNELS.rc.cordis, "4.0.6-alpha.1", "4.0.6-alpha.2"), + }; +})(); + +/** The next alpha cut: dsh moved, and its cluster's ranges open at alpha.2. */ +const ALPHA_3: RegistryView = (() => { + const next = { ...ALPHA_PATCHED, dsh: "0.2.2-alpha.3" }; + const declared = declaredBy(next); + return { + ...TODAY, + appBoot: { + versions: { ...TODAY.appBoot.versions, ...declared.appBoot }, + }, + bridge: { versions: { ...TODAY.bridge.versions, ...declared.bridge } }, dsh: { - "dist-tags": { alpha: "0.2.2-alpha.3", latest: CHANNELS.rc.dsh }, + "dist-tags": { alpha: next.dsh, latest: CHANNELS.rc.dsh }, + versions: { ...TODAY.dsh.versions, ...declared.dsh }, }, - loader: versions(CHANNELS.rc.loader, "1.0.7-alpha.1", "1.0.7-alpha.2"), - schemastery: versions( - CHANNELS.rc.schemastery, - "3.18.6-alpha.1", - "3.18.6-alpha.2" - ), }; })(); @@ -144,38 +192,99 @@ describe("host drift · the rules", () => { it("names a channel whose dist-tag moved, alone when nothing else did", () => { const registry = registryOf(CHANNELS); + // The next cut, declaring exactly the ranges the record's did. + const same = declaredBy({ ...CHANNELS.alpha, dsh: "0.2.2-alpha.3" }); const moved: RegistryView = { ...registry, - bridge: { - versions: { - ...registry.bridge.versions, - "0.2.2-alpha.3": registry.bridge.versions?.[CHANNELS.alpha.dsh] ?? {}, - }, + appBoot: { + versions: { ...registry.appBoot.versions, ...same.appBoot }, + }, + bridge: { versions: { ...registry.bridge.versions, ...same.bridge } }, + dsh: { + "dist-tags": { alpha: "0.2.2-alpha.3", latest: CHANNELS.rc.dsh }, + versions: { ...registry.dsh.versions, ...same.dsh }, }, - dsh: { "dist-tags": { alpha: "0.2.2-alpha.3", latest: CHANNELS.rc.dsh } }, }; expect(hostDrift(CHANNELS, moved)).toStrictEqual([ `alpha: @deepseek-ai/dsh@alpha is 0.2.2-alpha.3 on the registry; the record says ${CHANNELS.alpha.dsh}`, ]); }); - it("follows the cluster's ranges, not the packages' own tags: today's alpha cut", () => { + it("holds the family to the floors the cluster declares, not the newest patch inside them: today's registry", () => { + // Known answer: every member has a newer patch inside its `~` range, which + // the e2e-host npm cell runs; the record is the desktop's tree, the floor. + expect( + newestIn(versions("4.0.6-alpha.1", "4.0.6-alpha.2"), "~4.0.6-alpha.1") + ).toBe("4.0.6-alpha.2"); + expect(hostDrift(CHANNELS, TODAY)).toStrictEqual([]); + }); + + it("names every family floor a dsh cut moves, read from the package that declares it", () => { const lines = hostDrift(CHANNELS, ALPHA_3); - expect(lines).toHaveLength(4); - expect(lines[0]).toMatch( - /^alpha: @deepseek-ai\/dsh@alpha is 0\.2\.2-alpha\.3/u - ); - expect(lines).toContain( - `alpha: @deepseek-ai/cordis (~4.0.6-alpha.2) is 4.0.6-alpha.2 on the registry; the record says ${CHANNELS.alpha.cordis}` - ); - expect(lines).toContain( - `alpha: @deepseek-ai/cordis-plugin-loader (~1.0.7-alpha.2) is 1.0.7-alpha.2 on the registry; the record says ${CHANNELS.alpha.loader}` - ); - expect(lines).toContain( - `alpha: @deepseek-ai/schemastery (~3.18.6-alpha.2) is 3.18.6-alpha.2 on the registry; the record says ${CHANNELS.alpha.schemastery}` - ); - // The rc channel's records are current in this view, and stay silent. - expect(lines.filter((line) => line.startsWith("rc:"))).toStrictEqual([]); + const say = ( + name: string, + range: string, + found: string, + recorded: string + ) => + `alpha: the floor of ${name} ${range} is ${found} on the registry; the record says ${recorded}`; + expect(lines).toStrictEqual([ + `alpha: @deepseek-ai/dsh@alpha is 0.2.2-alpha.3 on the registry; the record says ${CHANNELS.alpha.dsh}`, + say( + "@deepseek-ai/cordis", + "~4.0.6-alpha.2", + "4.0.6-alpha.2", + CHANNELS.alpha.cordis + ), + say( + "@deepseek-ai/cordis-plugin-loader", + "~1.0.7-alpha.2", + "1.0.7-alpha.2", + CHANNELS.alpha.loader + ), + say( + "@deepseek-ai/schemastery", + "~3.18.6-alpha.2", + "3.18.6-alpha.2", + CHANNELS.alpha.schemastery + ), + say( + "@deepseek-ai/cordis-plugin-include", + "~1.0.11-alpha.2", + "1.0.11-alpha.2", + CHANNELS.alpha.include + ), + say( + "@deepseek-ai/cordis-plugin-timer", + "~1.1.8-alpha.2", + "1.1.8-alpha.2", + CHANNELS.alpha.timer + ), + say( + "@deepseek-ai/cordis-plugin-group", + "~1.0.6-alpha.2", + "1.0.6-alpha.2", + CHANNELS.alpha.group + ), + // cosmokit is declared by cordis, read at the cordis floor just found. + say( + "@deepseek-ai/cosmokit", + "~1.8.7-alpha.2", + "1.8.7-alpha.2", + CHANNELS.alpha.cosmokit + ), + ]); + }); + + it("checks the timer plugin, which no lockfile holds, against the published dsh", () => { + const registry = registryOf(CHANNELS); + const wrong = { + ...CHANNELS, + alpha: { ...CHANNELS.alpha, timer: "1.1.8-alpha.2" }, + }; + expect(hostDrift(wrong, registry)).toStrictEqual([ + `alpha: the floor of @deepseek-ai/cordis-plugin-timer ~${CHANNELS.alpha.timer} is ${CHANNELS.alpha.timer} on the registry; the record says 1.1.8-alpha.2`, + ]); }); it("reports a moved pi range by the range, its floor and what it resolves", () => { @@ -221,12 +330,11 @@ describe("host drift · the rules", () => { it("never reads 'nothing answered' as current", () => { expect( hostDrift(CHANNELS, { + appBoot: {}, bridge: {}, cordis: {}, dsh: {}, - loader: {}, pi: {}, - schemastery: {}, }) ).toStrictEqual([ "alpha: @deepseek-ai/dsh has no 'alpha' dist-tag; nothing compared", @@ -258,7 +366,21 @@ describe("host drift · the rules", () => { expect( hostDrift({ rc: CHANNELS.rc }, { ...registry, cordis: {} }) ).toStrictEqual([ - `rc: no published @deepseek-ai/cordis satisfies ~${CHANNELS.rc.cordis}`, + `rc: @deepseek-ai/cordis@${CHANNELS.rc.cordis} is not published; @deepseek-ai/cosmokit was not compared`, + ]); + expect( + hostDrift( + { rc: CHANNELS.rc }, + { + ...registry, + dsh: { "dist-tags": registry.dsh["dist-tags"] }, + appBoot: {}, + } + ) + ).toStrictEqual([ + `rc: @deepseek-ai/dsh@${CHANNELS.rc.dsh} is not published; @deepseek-ai/cordis-plugin-include was not compared`, + `rc: @deepseek-ai/dsh@${CHANNELS.rc.dsh} is not published; @deepseek-ai/cordis-plugin-timer was not compared`, + `rc: @deepseek-ai/dsh-app-boot@${CHANNELS.rc.dsh} is not published; @deepseek-ai/cordis-plugin-group was not compared`, ]); }); }); diff --git a/test/support/host-versions.ts b/test/support/host-versions.ts index cf39d086..4683bcea 100644 --- a/test/support/host-versions.ts +++ b/test/support/host-versions.ts @@ -3,12 +3,16 @@ * harness channels, as each one actually ships (D14'', D22''). * * - `rc` is `@deepseek-ai/dsh@latest`; `alpha` is `@deepseek-ai/dsh@alpha`. - * Each record names the `@deepseek-ai` cluster's tag, cordis and its loader - * (each on its own line, at what the cluster's `~` range resolves), the - * `@deepseek-ai/schemastery` the cluster's range resolves, and the rest of - * the cordis family the published host installs (`cordisFamily`): the - * versions the harness vendors at the channel's tag, which is the tree the - * desktop app ships and the one the `e2e-host` job's floor cell forces. + * Each record names the `@deepseek-ai` cluster's tag and the cordis family + * the published host installs (`cordisFamily`: cordis, its loader, include, + * group and timer plugins, cosmokit and schemastery), each at the FLOOR of + * the `~` range the published cluster declares for it. That floor is the + * version the harness vendors at the channel's tag, the tree the desktop app + * ships, what `package.json` pins and the lockfile holds, and what the + * `e2e-host` job's floor cell forces. A newer patch inside the same range is + * what an npm install resolves; only the e2e-host npm cell runs it, and it + * never moves the record: the family moves only with `dsh`, to the new + * tag's floors (`scripts/host-drift.ts` reads them off the registry). * - `piRange` is the range the bridge (`dsh-llm-pi-ai`) declares. Nothing in * the published packages narrows it: no shrinkwrap, no bundled copy, no * install script and no runtime version check, so an npm install runs pi @@ -41,19 +45,19 @@ import { join } from "node:path"; export type Channel = "alpha" | "rc"; export interface ChannelRecord { - /** `@deepseek-ai/cordis`, what the cluster's `~` range resolves. */ + /** `@deepseek-ai/cordis`, the floor of the bridge's `~` range. */ readonly cordis: string; - /** `@deepseek-ai/cosmokit`, cordis's own utility library. */ + /** `@deepseek-ai/cosmokit`, the floor of the range `cordis` declares. */ readonly cosmokit: string; /** The npm dist-tag the channel publishes `@deepseek-ai/dsh` under. */ readonly distTag: "alpha" | "latest"; /** The `@deepseek-ai/dsh-*` cluster's version. */ readonly dsh: string; - /** `@deepseek-ai/cordis-plugin-group`. */ + /** `@deepseek-ai/cordis-plugin-group`, the floor of dsh-app-boot's range. */ readonly group: string; - /** `@deepseek-ai/cordis-plugin-include`. */ + /** `@deepseek-ai/cordis-plugin-include`, the floor of dsh's range. */ readonly include: string; - /** `@deepseek-ai/cordis-plugin-loader`, what the cluster's range resolves. */ + /** `@deepseek-ai/cordis-plugin-loader`, the floor of the bridge's range. */ readonly loader: string; /** The committed lockfile that pins this channel, from the repo root. */ readonly lockfile: string; @@ -63,12 +67,14 @@ export interface ChannelRecord { readonly piFloor: string; /** The pi range the bridge, `dsh-llm-pi-ai`, declares. */ readonly piRange: string; - /** `@deepseek-ai/schemastery`, what the cluster's `~` range resolves. */ + /** `@deepseek-ai/schemastery`, the floor of the bridge's `~` range. */ readonly schemastery: string; /** - * `@deepseek-ai/cordis-plugin-timer`. The only family member this - * repository's own install does not resolve, so no lockfile holds it; the - * `e2e-host` job's floor cell asserts it against the published host's tree. + * `@deepseek-ai/cordis-plugin-timer`, the floor of `@deepseek-ai/dsh`'s + * range. The only family member this repository's own install does not + * resolve, so no lockfile holds it: the nightly drift report holds it to the + * range the published dsh declares, and the binary e2e to the range the + * installed dsh declares. */ readonly timer: string; } @@ -108,10 +114,10 @@ export const CHANNELS: Readonly> = { /** * The cordis family the published host installs, by package name, at the - * record's versions: cordis, its loader, include, group and timer plugins, - * cosmokit and schemastery. They are released together and peer on one - * another by `~` ranges, so forcing one of them alone (cordis, say) leaves a - * tree nobody ships. + * record's versions (the declared ranges' floors): cordis, its loader, + * include, group and timer plugins, cosmokit and schemastery. They are + * released together and peer on one another by `~` ranges, so forcing one of + * them alone (cordis, say) leaves a tree nobody ships. * * @param record - a channel's record. * @returns `{ "": "" }` for every family member. @@ -130,9 +136,12 @@ export const cordisFamily = ( /** * The npm `overrides` the `e2e-host` job's floor cell installs the published - * host with: the channel's cordis family at the record's versions and pi at - * the bridge range's floor, which is the tree the desktop app ships (its pi - * patched there, not here). The npm cell takes no override. + * host with: the channel's cordis family at the record's versions (the + * declared ranges' floors) and pi at the bridge range's floor, with the + * telemetry package pi ships beside it at the same version, which is the tree + * the desktop app ships (its pi patched there, not here). pi-ai `1.1.0` + * declares its telemetry `^1.1.0` and would float it; from `1.3.0` pi pins + * the pair exactly. The npm cell takes no override. * * @param record - a channel's record. * @returns the overrides, keyed by package name. @@ -142,6 +151,7 @@ export const floorOverrides = ( ): Readonly> => ({ ...cordisFamily(record), "@earendil-works/pi-ai": record.piFloor, + "@earendil-works/pi-telemetry": record.piFloor, }); /** The channel the root `package.json` and `pnpm-lock.yaml` install. */ diff --git a/test/support/host.ts b/test/support/host.ts index 7e883d59..86c9e284 100644 --- a/test/support/host.ts +++ b/test/support/host.ts @@ -501,10 +501,12 @@ export const CHARACTERIZED_MODELS: Readonly> = * - `KNOWN_DEFECT` is wrong today, and a phase fixes it; * - `INTENDED_CHANGE` is right today, and a phase changes it on purpose; * - `BASELINE` must survive every phase; - * - `HOST_LIMITATION` is something one host cell lacks or breaks, its pi or - * its own code, that no phase of this plugin fixes (D26: a host gap is the - * host's job): its title names the channel and the host it applies to, and - * it is reported upstream. + * - `HOST_LIMITATION` is something one channel's host, as one install runs + * it, lacks or breaks, its pi or its own code, that no phase of this plugin + * fixes (D26: a host gap is the host's job). The install is a host cell + * (`HostCell`) for an in-process row, or an `e2e-host` cell (`npm` or + * `floor`) for the binary e2e; the title names the channel, the host and + * that install, and it is reported upstream. */ export type RowTag = | "BASELINE" From ec38185d750412b6e520f94b176be8b3e076f1d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 01:26:59 +0000 Subject: [PATCH 11/11] fix: hold the cordis family at the record's floors through a relock A relock re-resolves include, group and cosmokit, which arrive only transitively, at the newest patch their `~` ranges admit; once the harness publishes one, the lockfile family case fails on the next routine dependency change. Measured: a fresh resolve of the alpha lockfile with the release-age window lifted takes the alpha.2 patches. - fixed: rules-docs-3-RD3-2 - package.json pins include, group and cosmokit like cordis, the loader and schemastery, and pinFor moves each with the channel; pnpm resolves those peers from the root pins and dedupes cosmokit onto its pin, so a fresh resolve now holds the floors (overrides were tried and rejected: pnpm rewrites the lockfile's peer ranges with them, which would make the peer-floor witness compare the record with itself). A new harness-channel case requires every resolved member to be a direct pin (failed on both lockfiles before), and use-channel.test.ts holds the new rules. Both lockfiles relocked; only importer entries changed. CONTRIBUTING (relock rule, release step 3), AGENTS.md and D22'' say a relock keeps the family at the floors. - fixed: rules-docs-3-RD3-5 - the host-versions.ts and host-drift.ts headers say package.json pins every member the install resolves (all but the timer plugin), so each lockfile holds the floors by construction. - fixed: both-kept-3-R3-1 - the harness-channel timer comment names the binary e2e and the drift report, not "the next case"; the cosmokit rule (checked only where the installed cordis is the record's, otherwise only by the drift report; today not on alpha's npm cell) is stated in CONTRIBUTING, plan step 1 and the e2e header. - fixed: resolutions-3-T1B-R3-4, rules-docs-3-RD3-4 - same edit; "each record value" narrowed to "each cordis-family field". - fixed: both-kept-3-R3-2, resolutions-3-T1B-R3-2, rules-docs-3-RD3-3 - scripts/host-drift.ts added to the e2e-host scope regex, with its reason in the comment, CONTRIBUTING's Triggers and the plan's "When it runs". - fixed: resolutions-3-T1B-R3-1, rules-docs-3-RD3-1 - the docs/testing.md row states the floor rule for all seven members and the known answers. - fixed: resolutions-3-T1B-R3-3 - the FAMILY_DECLARERS doc says the drift report reads the dist-tagged cluster and the e2e the record's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0141fKp5k6hiaJcP5ovhGtSD --- .github/workflows/ci.yml | 6 ++++-- AGENTS.md | 2 +- CONTRIBUTING.md | 8 ++++---- channels/rc/pnpm-lock.yaml | 9 +++++++++ docs/improvement-plan.md | 6 +++--- docs/testing.md | 2 +- package.json | 3 +++ pnpm-lock.yaml | 9 +++++++++ scripts/channel-manifest.ts | 22 ++++++++++++++++++++++ scripts/host-drift.ts | 20 ++++++++++++-------- test/e2e/host-binary.e2e.ts | 6 +++++- test/harness-channel.test.ts | 35 ++++++++++++++++++++++++++++++++--- test/support/host-versions.ts | 7 +++++-- test/use-channel.test.ts | 26 +++++++++++++++++++++++--- 14 files changed, 133 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47aeea56..53bcae92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -458,7 +458,9 @@ jobs: # loads that bundle into a real host is this one, and its externals # keep cordis and schemastery the host's copies) and the test # settings vitest.e2e.config.ts inherits. test/support/host-versions.ts - # names the binary each channel installs. package.json is not (every + # names the binary each channel installs, and scripts/host-drift.ts + # holds the family table the binary e2e asserts against the installed + # manifests. package.json is not (every # dependency bump touches it), so a change to its `test:e2e:host` # script alone is exercised only by the nightly run. # Until P-R1 moves the host seam under src/host/, the modules that @@ -471,7 +473,7 @@ jobs: # in-process characterization of the same rows, on both channels, and # here by the nightly run. if printf '%s\n' "$changed" \ - | grep -Eq '^(src/host/|src/lifecycle\.ts$|src/identity\.ts$|src/responses-provider\.ts$|src/responses-routes\.ts$|src/models-discovery\.ts$|src/google-bridge\.ts$|src/fetch-patch\.ts$|src/pi-provider\.ts$|src/stream-hook\.ts$|cordis\.patch\.yml$|test/e2e/host/|test/e2e/host-binary\.e2e\.ts$|test/support/host\.ts$|test/support/host-versions\.ts$|\.github/workflows/ci\.yml$|vitest\.e2e\.config\.ts$|vite\.config\.ts$)'; then + | grep -Eq '^(src/host/|src/lifecycle\.ts$|src/identity\.ts$|src/responses-provider\.ts$|src/responses-routes\.ts$|src/models-discovery\.ts$|src/google-bridge\.ts$|src/fetch-patch\.ts$|src/pi-provider\.ts$|src/stream-hook\.ts$|cordis\.patch\.yml$|test/e2e/host/|test/e2e/host-binary\.e2e\.ts$|test/support/host\.ts$|test/support/host-versions\.ts$|scripts/host-drift\.ts$|\.github/workflows/ci\.yml$|vitest\.e2e\.config\.ts$|vite\.config\.ts$)'; then echo "run=true" >> "$GITHUB_OUTPUT" else echo "run=false" >> "$GITHUB_OUTPUT" diff --git a/AGENTS.md b/AGENTS.md index a7d96723..eb2e3606 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,7 +47,7 @@ Host bundle (`lib/index.mjs`), a thin `apply` barrel over small modules: - `test/support/keyless-matrix.ts` — the keyless 401-vs-404 probe matrix's cell table and classifier. Pure, though it lives in `test/support/` rather than `src/utils/`: a function of its arguments, with no code path that can set a credential header; `test/e2e/keyless-matrix.e2e.ts` sends the cells. - `test/e2e/host/` — the measurement bundle (`probe-plugin.mjs`, its manifest and patch) the `e2e-host` job links into a throwaway profile under a temporary HOME, through a relative `link:` path. `test/e2e/host-binary.e2e.ts` is its reader, gated on `DSH_HOST_E2E=1`; it installs no host itself and needs `DSH_HOST_PREFIX`, a prefix holding the installed one. - `scripts/report-host-drift.ts` — the nightly harness-drift report (CI's `host-drift` job, schedule and dispatch only): reads the registry the way the records are defined (`dsh` at its dist-tag, pi's range, floor and newest, and every cordis-family member at the floor of the range the tagged cluster declares) and prints one `::warning::` per compared field that moved, exiting 0 whatever it finds, because a channel moving is the harness's release, not a regression. Its rules are `scripts/host-drift.ts` (`hostDrift`, pure), held by `test/host-drift.test.ts` with known answers, a loopback registry and the job's evaluated condition. P-T3 turns the report into a proposed bump pull request. -- `scripts/use-channel.ts` — switches the dev install: `rc` or `alpha` installs that channel's committed lockfile (unpatched pi at the newest version the bridge's range admits, what an npm user runs), and `--floor` the same cluster with pi at the range's floor, never committed. It restores every tracked file it touched, so a switch leaves no diff. Its pure half is `scripts/channel-manifest.ts` (`channelManifest`, `pinFor`, `workspaceWithPi`, and `withoutHostPackages`, the lockfile `rc --relock` starts from), held by `test/use-channel.test.ts`. `rc --relock` belongs in the same commit as ANY change to `package.json`'s dependencies or `pnpm-workspace.yaml`'s catalog or overrides, not only a channel bump: the rc lockfile carries every specifier the root one does and CI installs it frozen, and `test/harness-channel.test.ts` fails, naming the command, when they disagree. Dependabot ignores the cluster and pi, which move only as a channel bump. **This repository carries no pi patch** (D14''): an npm install runs pi unpatched, the harness's patch reaches users only through the desktop app and touches nothing the plugin reads, and a version-keyed patch would break every pi bump. `test/architecture.test.ts` keeps it that way by failing if `src` ever reads a streamed tool-call delta, partial arguments or a thought signature. +- `scripts/use-channel.ts` — switches the dev install: `rc` or `alpha` installs that channel's committed lockfile (unpatched pi at the newest version the bridge's range admits, what an npm user runs), and `--floor` the same cluster with pi at the range's floor, never committed. It restores every tracked file it touched, so a switch leaves no diff. Its pure half is `scripts/channel-manifest.ts` (`channelManifest`, `pinFor`, `workspaceWithPi`, and `withoutHostPackages`, the lockfile `rc --relock` starts from), held by `test/use-channel.test.ts`. A relock keeps the cordis family at the record's floors because `package.json` pins every member the install resolves (include, group and cosmokit too, which arrive transitively and would otherwise resolve to the newest patch inside their `~` ranges) and `pinFor` moves each with the channel. `rc --relock` belongs in the same commit as ANY change to `package.json`'s dependencies or `pnpm-workspace.yaml`'s catalog or overrides, not only a channel bump: the rc lockfile carries every specifier the root one does and CI installs it frozen, and `test/harness-channel.test.ts` fails, naming the command, when they disagree. Dependabot ignores the cluster and pi, which move only as a channel bump. **This repository carries no pi patch** (D14''): an npm install runs pi unpatched, the harness's patch reaches users only through the desktop app and touches nothing the plugin reads, and a version-keyed patch would break every pi bump. `test/architecture.test.ts` keeps it that way by failing if `src` ever reads a streamed tool-call delta, partial arguments or a thought signature. - `src/config.ts` · `src/config-values.ts` · `src/utils/guards.ts` — the schema and its defaults; dependency-free readers and type guards. The client bundle may import these, and it does — along with the contract and presentation modules (`usage-contract.ts`, `settings-copy.ts`, `settings-fields.ts`, `usage-ui.ts`, `go-limits.ts`, `opencode-store-contract.ts`). The rule is not a list of names: **the client may import a module that reaches nothing host-only** — no cordis context, no credential seam, no node built-in — and the bundle gate is what enforces it, by failing the build rather than by anyone remembering. An earlier version of this line named two files as the only ones allowed, which the tree had already outgrown. ## `src/utils/` — pure by construction, enforced by test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45f9be41..21c09f20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ ## Prerequisites - Node.js `>= 24` (CI runs the version in `.node-version`, and the floor in a job of its own), `pnpm` (v12). The suite refuses to start below the floor and names both versions: below it, `node:sqlite`, `Intl.DurationFormat` and the ICU strings the tests pin differ, and the failures read like regressions. -- The plugin supports the latest release of **both** harness channels, `rc` (`dsh@latest`) and `alpha` (`dsh@alpha`). Their hosts (the `@deepseek-ai` cluster, cordis, its loader, and the pi range the bridge declares) are named once, in `test/support/host-versions.ts`. `pnpm install` installs the alpha channel from the root lockfile; `node --experimental-strip-types scripts/use-channel.ts rc` switches to the rc channel's committed lockfile (`channels/rc/pnpm-lock.yaml`), `… alpha` switches back, and `--floor` puts pi at the range's floor (the harness's own pin) for a local run. Each install runs pi **unpatched**, as an npm user does: this repository carries no pi patch. A switch restores every file it touched, so `git status` stays clean, and the suite reads which channel is installed from the marker the switch leaves (`DSH_CHANNEL` overrides it). `scripts/use-channel.ts rc --relock` regenerates the rc lockfile from the root one, and it runs in the same commit as **any** change to `package.json`'s dependencies or to `pnpm-workspace.yaml`'s catalog or overrides, not only a channel bump: the rc lockfile carries every specifier the root one does, CI installs it frozen, and `test/harness-channel.test.ts` fails, naming the command, when the two disagree (a Dependabot pull request included, which cannot relock it). +- The plugin supports the latest release of **both** harness channels, `rc` (`dsh@latest`) and `alpha` (`dsh@alpha`). Their hosts (the `@deepseek-ai` cluster, cordis, its loader, and the pi range the bridge declares) are named once, in `test/support/host-versions.ts`. `pnpm install` installs the alpha channel from the root lockfile; `node --experimental-strip-types scripts/use-channel.ts rc` switches to the rc channel's committed lockfile (`channels/rc/pnpm-lock.yaml`), `… alpha` switches back, and `--floor` puts pi at the range's floor (the harness's own pin) for a local run. Each install runs pi **unpatched**, as an npm user does: this repository carries no pi patch. A switch restores every file it touched, so `git status` stays clean, and the suite reads which channel is installed from the marker the switch leaves (`DSH_CHANNEL` overrides it). `scripts/use-channel.ts rc --relock` regenerates the rc lockfile from the root one, and it runs in the same commit as **any** change to `package.json`'s dependencies or to `pnpm-workspace.yaml`'s catalog or overrides, not only a channel bump: the rc lockfile carries every specifier the root one does, CI installs it frozen, and `test/harness-channel.test.ts` fails, naming the command, when the two disagree (a Dependabot pull request included, which cannot relock it). A relock keeps the cordis family at the record's floors: `package.json` pins every member the install resolves (cordis, the loader, include, group, cosmokit, schemastery; the timer plugin is resolved by nothing here) and `pinFor` moves each with the channel, so pnpm resolves include and group (peers it installs itself) and cosmokit (a dependency of cordis) from those pins rather than taking the newest patch inside their `~` ranges. A relock that fails the family case in `test/harness-channel.test.ts` means a member arrived some other way: pin it the same way, never loosen the case. - A checkout of this repo; for live testing, a DeepSeek Harness profile (see README for the `link:` setup) ## Commands @@ -79,9 +79,9 @@ What every case does: `.github/workflows/ci.yml`'s `e2e-host` job installs each channel's published `@deepseek-ai/dsh` (its record's `dsh`) into a throwaway prefix with npm, the way a user installs it, builds this checkout, and runs `test/e2e/host-binary.e2e.ts`. That file creates a temporary HOME whose profile links this checkout and the probe bundle (`test/e2e/host/`) through relative `link:` paths, stores dummy keys in the temporary home's credentials document (`$DSH_HOME/.credentials.yaml`, shared by every profile under that home), boots `dsh --profile e2e --no-open --port 0`, and reads what `test/e2e/host/probe-plugin.mjs` measured. It asserts today's rows; P-R1 flips them. -- **Cells, per channel, pi unpatched in both.** `npm` is the published host exactly as npm resolves it, cordis and pi included, what an npm user runs, and asserts the `newest` rows while its pi is the channel record's; a pi published since the record still has to sit in the bridge's range, but the rows that depend on which ids pi describes (the keyed listing and the per-model rows) are skipped with a warning, and the nightly `host-drift` job names the new pi. `floor` is the tree the desktop app ships: npm `overrides` (`floorOverrides` in `host-versions.ts`) force the channel's whole cordis family (cordis, the loader, the include, group and timer plugins, cosmokit, schemastery) at the record's versions, which are the floors of the `~` ranges the published cluster declares (what the harness vendors at the tag), and pi with its telemetry package at the range's floor; the job and the e2e assert that npm applied each one, and the cell asserts every row of the channel's floor cell. In both cells the e2e also holds each record value to the floor of the range the installed published package declares for it, which no override rewrites. It is cached by channel, `dsh` and a hash of those overrides, so a cache filled before the record or its overrides moved is never reused; the rest of the tree is what npm resolved when the cache was filled. The installed `dsh` must be the channel record's, so a prefix of the wrong channel fails by name. The save rows run whatever pi resolved (they do not depend on which ids pi describes, though they assume pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage that as a pi change first); they skip only where the host itself cannot save (next bullet). +- **Cells, per channel, pi unpatched in both.** `npm` is the published host exactly as npm resolves it, cordis and pi included, what an npm user runs, and asserts the `newest` rows while its pi is the channel record's; a pi published since the record still has to sit in the bridge's range, but the rows that depend on which ids pi describes (the keyed listing and the per-model rows) are skipped with a warning, and the nightly `host-drift` job names the new pi. `floor` is the tree the desktop app ships: npm `overrides` (`floorOverrides` in `host-versions.ts`) force the channel's whole cordis family (cordis, the loader, the include, group and timer plugins, cosmokit, schemastery) at the record's versions, which are the floors of the `~` ranges the published cluster declares (what the harness vendors at the tag), and pi with its telemetry package at the range's floor; the job and the e2e assert that npm applied each one, and the cell asserts every row of the channel's floor cell. In both cells the e2e also holds each cordis-family field to the floor of the range the installed published package declares for it, which no override rewrites, with one rule for cosmokit: its declarer is cordis, so it is checked only where the installed cordis is the record's, and elsewhere only the nightly drift report checks it (today that leaves it out of alpha's `npm` cell, where cordis floats to a newer patch). It is cached by channel, `dsh` and a hash of those overrides, so a cache filled before the record or its overrides moved is never reused; the rest of the tree is what npm resolved when the cache was filled. The installed `dsh` must be the channel record's, so a prefix of the wrong channel fails by name. The save rows run whatever pi resolved (they do not depend on which ids pi describes, though they assume pi lacks at least one of `ZEN_UNDESCRIBED` and still describes every other declared Zen id, so a pi that learns all of them or retires a declared id turns them red by design: triage that as a pi change first); they skip only where the host itself cannot save (next bullet). - **The host's own failures are recorded, not worked around.** Only the `npm` cell lets cordis float, as the published host's `~` ranges do for a user, and the job reports what it resolved. Where that leaves the host unable to run its own save path, the run records a `[HOST_LIMITATION]` row (titled with the channel and the `dsh`) and skips the save rows. It is detected from what the host answered, never from a version (D26): the plane edit and a control edit no plugin's listener reads (the probe's own entry) both fail with the same `TypeError` thrown by `@deepseek-ai/dsh-app-boot`; where the host saves, the row skips and a `[BASELINE]` row asserts the control was accepted. Today that is alpha's `0.2.2-alpha.2` as npm installs it: cordis `4.0.6-alpha.2` calls `internal/update` in a new shape while `dsh-app-boot` registers its listener in the old one, so every config-editor save throws `next is not a function`. The alpha floor, on the cordis the harness vendors at the tag, saves. -- **Triggers.** Dispatch and the nightly run take both cells of both channels. A pull request takes the `npm` cell of both channels, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` (what the job installs), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. +- **Triggers.** Dispatch and the nightly run take both cells of both channels. A pull request takes the `npm` cell of both channels, and only when it touches `src/host/**`, `src/lifecycle.ts`, `src/identity.ts`, the modules that install the host patches and mounts whose behaviour depends on the real loader and apply order, until P-R1 moves them under `src/host/` (`src/responses-provider.ts`, `src/responses-routes.ts`, `src/models-discovery.ts`, `src/google-bridge.ts`, `src/fetch-patch.ts`, `src/pi-provider.ts`, `src/stream-hook.ts`; every other row producer, such as `src/key-capture.ts`, `src/models-catalog.ts` or `src/config.ts`, is covered on each pull request by the in-process characterization of the same `S0_TODAY` rows, and by the nightly run), `cordis.patch.yml`, `test/e2e/host/**`, `test/e2e/host-binary.e2e.ts`, `test/support/host.ts` (the rows the binary e2e asserts), `test/support/host-versions.ts` (what the job installs), `scripts/host-drift.ts` (the family table the binary e2e asserts against the installed manifests), `.github/workflows/ci.yml`, `vitest.e2e.config.ts` or `vite.config.ts` (the job's own steps and runner config; `vite.config.ts` also holds the pack config that builds `lib/index.mjs`, the bundle only this job loads into a real host, and the test settings the e2e config inherits) (scoped by a step: a workflow-level path filter would silence every other job; a diff the step cannot compute fails it rather than skipping the job). `package.json` is left out because every dependency bump touches it, so a change to its `test:e2e:host` script alone is exercised only by the nightly run. It needs no secret. To run it locally, install a channel's host into a prefix outside the checkout, then point the suite at it (placeholder paths; `pnpm` must be on `PATH`). The first line reads the channel's record from `test/support/host-versions.ts`, the same way the CI job does, so the snippet never names a version itself: @@ -161,7 +161,7 @@ git diff --name-only main... \ 1. Use conventional commits (`feat:`, `fix:`) — release-please opens the version-bump + changelog PR automatically; merging it tags and creates the GitHub Release, which fires OIDC publishing to npm and GitHub Packages. 2. Avoid manual releases: release-please anchors on the latest GitHub **Release**, not the latest tag, so a bare tag is invisible to it (`AGENTS.md`, "Releasing"). If one is unavoidable, bump `version` in `package.json`, commit, tag, push the tag, and create the GitHub Release in the same step (`gh release create vX.Y.Z --generate-notes`). Never touch `CHANGELOG.md`: release-please owns it. -3. On every release, after smoke-testing (registry install resolves, plugin loads in the DSH Web profile, and a free-tier Zen call succeeds), check the **Supported: the latest release of both DeepSeek Harness channels** line and the **Support policy** paragraph under "Compatibility & Verification" in both READMEs. Their "today" versions, pi ranges and floors, and the default install named a paragraph below, are copied from the channel records in `test/support/host-versions.ts`, and `test/harness-channel.test.ts` holds them to the records. A channel bump **inside the prerelease tuples the peer ranges already admit** (a new `dsh` cut whose cluster changes no host hook, say) moves version data, and this is its whole set: for `rc`, the rc record and `channels/rc/pnpm-lock.yaml` (`scripts/use-channel.ts rc --relock`); for `alpha`, the alpha record, `package.json`'s exact `@deepseek-ai/*` pins and the root lockfile (`pnpm install`); for either, those README sentences in both languages, and `minimumReleaseAgeExclude` lines in `pnpm-workspace.yaml` for the exact versions when the cut is younger than pnpm's one-day release-age window. A pi move is the same data: `pnpm install` and `--relock` keep the pi a lockfile already satisfies, so move it with `pnpm update @earendil-works/pi-ai --no-save` (the root lockfile; `package.json` keeps the union range), then `scripts/use-channel.ts rc --relock`, which carries the root's pi into the rc lockfile, and set each record's `pi`. The cordis family (cordis, the loader, include, group, timer, cosmokit, schemastery) moves only with `dsh`, to the floors of the `~` ranges the new tag's cluster declares (the drift report prints them; they are what the harness vendors at that tag): a newer family patch inside the same ranges is what the `e2e-host` npm cell already runs and logs, never a record bump. The same update is what leaves one pi when a new bridge raises its floor above the locked one. The F10 characterization (`test/mount-protocols.test.ts`) is keyed by pi line, so a pi inside a line leaves it alone; a pi outside every line is a new major, and a reviewed change. **Data only while the host behaves the same.** A bump stays version data while tiers A and B and the binary e2e stay green on the new install. A pull request runs only the default installs, so a bump that moves `piFloor` or any cordis-family field is green only after a `workflow_dispatch` run on the bump branch, which adds both channels' floor cells to the `test` and `e2e-host` jobs. One that turns them red because the host behaves differently is a reviewed change made in the bump's own pull request and named in its body: a pi whose catalog or bases moved (a model added or dropped, a gap id learned) re-measures the `newest` cell's tables (`WHOLE_CATALOG` in `test/host-shapes.test.ts`, `PI_GAPS_BY_CELL` and `S0_NEWEST` in `test/support/host.ts`), and a moved `piFloor` re-measures that floor's tables the same way (`S0_RC_FLOOR` or `S0_ALPHA_FLOOR`, and the floor's columns of `WHOLE_CATALOG` and `PI_GAPS_BY_CELL`), and a host event whose call shape or waterfall order moved adapts the tier listeners in `test/support/host.ts` and the contract tests, re-measuring the facts they pin (F8, F12, F14, F20) rather than accepting both shapes. The next alpha cut is that case: cordis `4.0.6-alpha.2` changed the call shape of `internal/config` and `internal/update`, so the `0.2.2-alpha.3` bump rewrites the tier listeners (a follow-up to the restack). A cut that **opens a new prerelease tuple** (dsh `0.2.3-*`, cordis `4.0.7-*`) is not data: semver admits a prerelease only through a comparator in its own tuple, so it is a reviewed range change that also adds the matching `>=X.Y.Z-0 =X.Y.Z-0 > = + { + "@deepseek-ai/cordis-plugin-group": "group", + "@deepseek-ai/cordis-plugin-include": "include", + "@deepseek-ai/cosmokit": "cosmokit", + }; + const PI = "@earendil-works/pi-ai"; /** @@ -35,6 +53,10 @@ export const pinFor = (name: string, pins: ChannelPins): string | undefined => { if (name === "@deepseek-ai/cordis-plugin-loader") { return pins.loader; } + const member = FAMILY_PINS[name]; + if (member !== undefined) { + return pins[member]; + } return name.startsWith("@deepseek-ai/dsh-") ? pins.dsh : undefined; }; diff --git a/scripts/host-drift.ts b/scripts/host-drift.ts index 733b2ab7..610cbf1e 100644 --- a/scripts/host-drift.ts +++ b/scripts/host-drift.ts @@ -13,9 +13,11 @@ * same tuple), and `piFloor` is its floor; * - each member of the cordis family is the FLOOR of the `~` range the * published cluster declares for it (`FAMILY_DECLARERS`): the version the - * harness vendors at the tag, which the desktop app ships, `package.json` - * pins and the lockfile holds. A newer patch inside the same range is what - * the `e2e-host` job's npm cell runs and logs, never a record bump, so the + * harness vendors at the tag and the desktop app ships. `package.json` pins + * every member this repository's install resolves (all but the timer plugin, + * which nothing here depends on), so each lockfile holds them at that floor + * by construction. A newer patch inside the same range is what the + * `e2e-host` job's npm cell runs and logs, never a record bump, so the * family moves only when `dsh` does. * * Every function here is a function of its arguments - the registry's answers @@ -74,11 +76,13 @@ export type RegistryRole = keyof typeof REGISTRY_PACKAGES; export type RegistryView = Readonly>; /** - * Which published package declares each family member's range. The cluster's - * packages are read at the record's `dsh`; cosmokit is declared by cordis, - * read at the cordis floor. Cordis comes first, so its floor is known by the - * time cosmokit is read. The binary e2e reads the same table off the - * installed host. + * Which published package declares each family member's range. The drift + * report reads each cluster declarer at the `dsh` the channel's dist-tag names + * today (not the record's, so a new cut's floors are reported before the + * record moves), and cosmokit's declarer, cordis, at the cordis floor it read + * just before: cordis comes first, so its floor is known by the time cosmokit + * is read. The binary e2e reads the same table off the installed host, which + * is the record's `dsh`. */ export const FAMILY_DECLARERS: readonly { readonly declarer: "appBoot" | "bridge" | "cordis" | "dsh"; diff --git a/test/e2e/host-binary.e2e.ts b/test/e2e/host-binary.e2e.ts index 5611315f..d8c3ba99 100644 --- a/test/e2e/host-binary.e2e.ts +++ b/test/e2e/host-binary.e2e.ts @@ -45,7 +45,11 @@ * In both cells the record's family is held to something the record does not * produce: each member must be the floor of the `~` range the installed * published package declares for it (`FAMILY_DECLARERS`; overrides rewrite - * what is installed, never a manifest's declared range). + * what is installed, never a manifest's declared range). One rule narrows it: + * cosmokit's declarer is cordis, so cosmokit is checked only where the + * installed cordis is the record's; elsewhere only the nightly drift report + * checks it. Today that leaves it out of alpha's `npm` cell, where cordis + * floats to a newer patch. * * Only the `npm` cell lets cordis float, as the published host's `~` ranges * do for a user. Where what it resolved leaves the host unable to run its own diff --git a/test/harness-channel.test.ts b/test/harness-channel.test.ts index 5c543d93..dbd3f565 100644 --- a/test/harness-channel.test.ts +++ b/test/harness-channel.test.ts @@ -474,8 +474,8 @@ describe.each(Object.keys(CHANNELS) as Channel[])( // `cordisFamily` is what the `e2e-host` job forces on the floor cell, so // a family member here and the record disagreeing would make that cell // run a tree this install never measured. The timer plugin is the one - // member nothing here depends on, so no lockfile resolves it; the next - // case and the drift report hold it instead. + // member nothing here depends on, so no lockfile resolves it; the + // binary e2e and the nightly drift report hold it instead. const lock = lockfile(record.lockfile); const family = Object.entries(cordisFamily(record)); expect(family).toHaveLength(7); @@ -492,6 +492,33 @@ describe.each(Object.keys(CHANNELS) as Channel[])( }); }); + it("holds the family at the record's floors by construction: every member it resolves is a direct pin", () => { + // The case above holds every resolved member at the record's floor. + // A member that arrives only transitively (include and group as peers + // pnpm installs itself, cosmokit as a dependency of cordis) is resolved + // afresh by a relock or a regenerated lockfile at the NEWEST patch its + // `~` range admits, so the first patch published inside a range would + // turn the next routine dependency change red. A direct exact pin is + // what pnpm resolves those peers from and dedupes that dependency onto, + // so the root importer names every member the lockfile holds, and + // `pinFor` moves each with the channel. + const lock = lockfile(record.lockfile); + const family = Object.entries(cordisFamily(record)).filter( + ([name]) => lock.versionsOf(name).length > 0 + ); + // Known answer: all but the timer plugin, which nothing here resolves. + expect(family.map(([name]) => name)).not.toContain( + "@deepseek-ai/cordis-plugin-timer" + ); + expect(family).toHaveLength(6); + for (const [name, version] of family) { + expect(lock.direct.get(name), `${name} is pinned directly`).toBe( + version + ); + expect(pinFor(name, record), name).toBe(version); + } + }); + it("names each family member at the floor of the range the published cluster declares", () => { // The record is not its own witness: a family version must be the floor // of every `~` range the channel's published packages declare for it. @@ -501,7 +528,9 @@ describe.each(Object.keys(CHANNELS) as Channel[])( // desktop app ships; a newer patch inside the range is the npm cell's, // never the record's. cosmokit (a dependency of cordis) and the timer // plugin (declared by packages no lockfile here holds) are not peers - // here; the drift report and the binary e2e read them off the manifests. + // here; the drift report reads them off the published manifests, and + // the binary e2e off the installed ones (cosmokit only where the + // installed cordis is the record's). const { packages } = parse(lockfile(record.lockfile).project) as { packages: Record }>; }; diff --git a/test/support/host-versions.ts b/test/support/host-versions.ts index 4683bcea..d2fa13ea 100644 --- a/test/support/host-versions.ts +++ b/test/support/host-versions.ts @@ -8,8 +8,11 @@ * group and timer plugins, cosmokit and schemastery), each at the FLOOR of * the `~` range the published cluster declares for it. That floor is the * version the harness vendors at the channel's tag, the tree the desktop app - * ships, what `package.json` pins and the lockfile holds, and what the - * `e2e-host` job's floor cell forces. A newer patch inside the same range is + * ships, and what the `e2e-host` job's floor cell forces. `package.json` + * pins every member this repository's install resolves (all but the timer + * plugin, which nothing here depends on; `pinFor` moves each with the + * channel), so each committed lockfile holds them at that floor by + * construction, a relock included. A newer patch inside the same range is * what an npm install resolves; only the e2e-host npm cell runs it, and it * never moves the record: the family moves only with `dsh`, to the new * tag's floors (`scripts/host-drift.ts` reads them off the registry). diff --git a/test/use-channel.test.ts b/test/use-channel.test.ts index 37fb3539..ad2a2fa2 100644 --- a/test/use-channel.test.ts +++ b/test/use-channel.test.ts @@ -57,14 +57,34 @@ describe("channel switch · the manifest a channel installs", () => { }); it("refuses a harness package no rule covers instead of guessing its pin", () => { + // The timer plugin is the one family member nothing here resolves, so + // the manifest never pins it and no rule covers it. const stray = MANIFEST.replace( '"semver"', - '"@deepseek-ai/cosmokit": "1.8.7-alpha.1",\n "semver"' + '"@deepseek-ai/cordis-plugin-timer": "1.1.8-alpha.1",\n "semver"' ); expect(() => channelManifest(stray, CHANNELS.rc)).toThrow( - /@deepseek-ai\/cosmokit has no channel rule/u + /@deepseek-ai\/cordis-plugin-timer has no channel rule/u ); - expect(pinFor("@deepseek-ai/cosmokit", CHANNELS.rc)).toBeUndefined(); + expect( + pinFor("@deepseek-ai/cordis-plugin-timer", CHANNELS.rc) + ).toBeUndefined(); + }); + + it("moves the family members pinned only to hold their floors with the channel", () => { + // include, group and cosmokit arrive transitively; the manifest pins them + // so a relock cannot take a newer patch inside their `~` ranges. + for (const record of [CHANNELS.alpha, CHANNELS.rc]) { + expect(pinFor("@deepseek-ai/cordis-plugin-include", record)).toBe( + record.include + ); + expect(pinFor("@deepseek-ai/cordis-plugin-group", record)).toBe( + record.group + ); + expect(pinFor("@deepseek-ai/cosmokit", record)).toBe(record.cosmokit); + } + // Known answer: the two records differ, so the rule reads the record. + expect(CHANNELS.rc.cosmokit).not.toBe(CHANNELS.alpha.cosmokit); }); });