diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09a1bd8..7b3bd2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -371,17 +371,95 @@ jobs: # The live gateway contract: the only job that talks to OpenCode itself, so a # vendor payload change fails here rather than in a user's meter. # - # Secrets are not exposed to fork or Dependabot PRs, and every credentialed - # case skips when its key is absent — so this stays green there while still - # checking that the endpoints answer. Add OPENCODE_API_KEY (Zen) and - # OPENCODE_GO_API_KEY (Go plan) under Settings → Secrets → Actions to arm the - # keyed cases. + # What may spend a key is a tier (OPENCODE_E2E_KEYED), decided per event by + # the `tier` step: + # + # - `none`: spends no key, so only the keyless cases run — the keyless + # matrix, the keyless free-model probe, the System One family's three + # shapes (recorded), and the keyless cases of protocol-routing and + # opencode-live. Every pull request that does not + # touch the scope below, and a push to main. + # - `probes`: adds the minimal keyed probes (keyed-probes.e2e.ts), one + # request of a few tokens per plane × protocol × URL shape, plus six + # recorded variants of each plane's default completions cell (request + # id, project, reasoning effort; KEYED_VARIANTS) and every 429's + # Retry-After. A pull request + # that touches src/host/**; src/utils/protocols.ts, which decides the + # legacy URLs the probes send and the protocol each probe model speaks; + # src/utils/plane-model.ts, which decides the family the plugin + # dispatches to (AUTO_ENDPOINT_FAMILY), which the probes must keep + # matching; src/identity.ts, whose CLI version is in the user agent the + # probes carry; src/utils/request-shape.ts, the anthropic-version every + # Messages probe sends; or the probes' own inputs. Nothing else arms + # them. Left out, and covered by the offline suite and the nightly run, + # which arms everything: the generated catalog (src/catalog-data.ts) and + # the live model choice (src/models-catalog.ts), which pick the probes' + # modern URLs and models and whose paths the keyless matrix checks on + # every pull request; and the OPENCODE_UA alias in src/session.ts, which + # a turn and a probe share, so test/keyed-probes.test.ts holds the + # probe's identity to the real fetch patch offline. + # - `full`: adds the full keyed suites (protocol-routing, opencode-live, + # dispatch-levels). Nightly and on dispatch. + # + # The keys are passed only when the tier is not `none`, and the tier is + # always set explicitly: unset means `full` (so a local run with keys arms + # everything), and a dropped env line would arm the full suites on every + # pull request that arms the probes. Secrets are not exposed to fork or Dependabot PRs, and every + # credentialed case skips when its key is absent, so this stays green there + # while still checking that the endpoints answer. Add OPENCODE_API_KEY (Zen) + # and OPENCODE_GO_API_KEY (Go plan) under Settings → Secrets → Actions to arm + # the keyed cases. + # + # Both probe suites write a dated table of what each URL shape answered + # (docs/redirect-planes.md §13); the job appends them to the run summary and + # uploads them as the `endpoint-families` artifact. e2e: name: e2e · live gateway runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: which keyed tier does this run arm? + id: tier + env: + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + run: | + set -euo pipefail + case "$EVENT" in + schedule|workflow_dispatch) tier=full ;; + pull_request) + # Its own command, so a diff that cannot be computed fails the + # step rather than reading as "nothing changed". + changed="$(git diff --name-only "origin/${BASE_REF}...HEAD")" + # The listed modules, and nothing else: the host seam, the + # legacy URLs and protocols the probes send (protocols.ts), the + # family the plugin dispatches to (plane-model.ts), the CLI + # version in the user agent the probes carry (identity.ts), the + # anthropic-version every Messages probe sends + # (request-shape.ts), and the probes' own inputs, CI arming + # included. Left out, and covered by the offline suite and the + # nightly `full` run: the generated catalog (catalog-data.ts) + # and the live model choice (models-catalog.ts), which the + # keyless matrix checks on every pull request, and the + # OPENCODE_UA alias (session.ts), which the turn and the probe + # share and test/keyed-probes.test.ts holds offline. + # test/keyed-probes.test.ts compiles this pattern and checks + # it. grep reads a here-string, not a pipe: under + # pipefail, `grep -q` exiting on an early match could kill the + # writer with SIGPIPE on a long path list and read as a miss. + if grep -Eq '^(src/host/|src/utils/protocols\.ts$|src/utils/plane-model\.ts$|src/identity\.ts$|src/utils/request-shape\.ts$|test/e2e/keyed-probes\.e2e\.ts$|test/support/keyed-probes\.ts$|test/support/keyless-matrix\.ts$|\.github/workflows/ci\.yml$)' <<<"$changed"; then + tier=probes + else + tier=none + fi + ;; + *) tier=none ;; + esac + echo "keyed tier: $tier" + echo "keyed=$tier" >> "$GITHUB_OUTPUT" - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: @@ -391,8 +469,29 @@ jobs: - run: pnpm run test:e2e env: OPENCODE_E2E: "1" - OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} - OPENCODE_GO_API_KEY: ${{ secrets.OPENCODE_GO_API_KEY }} + OPENCODE_E2E_KEYED: ${{ steps.tier.outputs.keyed }} + OPENCODE_E2E_RECORD_DIR: ${{ runner.temp }}/endpoint-families + OPENCODE_API_KEY: ${{ steps.tier.outputs.keyed != 'none' && secrets.OPENCODE_API_KEY || '' }} + OPENCODE_GO_API_KEY: ${{ steps.tier.outputs.keyed != 'none' && secrets.OPENCODE_GO_API_KEY || '' }} + - name: publish the endpoint-family record to the run summary + if: always() + env: + RECORD_DIR: ${{ runner.temp }}/endpoint-families + run: | + for file in keyless.md keyed.md; do + if [ -f "$RECORD_DIR/$file" ]; then + cat "$RECORD_DIR/$file" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + fi + done + - name: upload the endpoint-family record + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: endpoint-families + path: ${{ runner.temp }}/endpoint-families + if-no-files-found: ignore + retention-days: 30 # The published host binary with this checkout linked into a throwaway # profile: the only run that boots `dsh` itself rather than the host packages @@ -486,8 +585,8 @@ jobs: # generated catalog) is covered on each pull request by the # in-process characterization of the same rows, on both channels, # and here by the nightly run. - if printf '%s\n' "$changed" \ - | grep -Eq '^(src/host/|src/lifecycle\.ts$|src/identity\.ts$|src/cordis-context\.ts$|src/models-discovery\.ts$|src/fetch-patch\.ts$|src/stream-hook\.ts$|cordis\.patch\.yml$|test/e2e/host/|test/e2e/host-binary\.e2e\.ts$|test/support/host\.ts$|test/support/host-versions\.ts$|scripts/host-drift\.ts$|\.github/workflows/ci\.yml$|vitest\.e2e\.config\.ts$|vite\.config\.ts$)'; then + # A here-string, not a pipe: see the e2e job's tier step. + if grep -Eq '^(src/host/|src/lifecycle\.ts$|src/identity\.ts$|src/cordis-context\.ts$|src/models-discovery\.ts$|src/fetch-patch\.ts$|src/stream-hook\.ts$|cordis\.patch\.yml$|test/e2e/host/|test/e2e/host-binary\.e2e\.ts$|test/support/host\.ts$|test/support/host-versions\.ts$|scripts/host-drift\.ts$|\.github/workflows/ci\.yml$|vitest\.e2e\.config\.ts$|vite\.config\.ts$)' <<<"$changed"; then echo "run=true" >> "$GITHUB_OUTPUT" else echo "run=false" >> "$GITHUB_OUTPUT" diff --git a/AGENTS.md b/AGENTS.md index 2f5e7ae..2f28a86 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,9 @@ Host bundle (`lib/index.mjs`), a thin `apply` barrel over small modules: - `test/support/src.ts` — the one door tests use into `src`: each name re-exported from the module that DEFINES it, never through the barrel, so a module move edits one file. `test/architecture.test.ts` enforces both halves — every name is declared in its target, and no test outside the named exemptions (`vi.mock` targets, fresh re-imports after `vi.resetModules()`, the architecture and public-API tests, the jsdom client tests, the `reset-process-state.ts` setup file) imports a `src` path, and no jsdom test loads the door. - `test/support/host-versions.ts` — the hosts the plugin supports, named once: one record per harness channel (`rc` = `dsh@latest`, `alpha` = `dsh@alpha`), each with its `@deepseek-ai` cluster, the cordis family (`cordisFamily`: cordis, the loader, include, group, timer, cosmokit, schemastery, each at the floor of the `~` range the published cluster declares, which is what the harness vendors at the tag and moves only with `dsh`), the bridge's pi RANGE, the newest pi that range resolves (the channel lockfile's) and the range's floor (the harness's own pin, which only the desktop app ships), and `floorOverrides`, the family and the floor pi pair the `e2e-host` floor cell forces, plus which one the install under test is (`DSH_CHANNEL`, else the marker `scripts/use-channel.ts` leaves, else `alpha`). `test/host-versions.test.ts` checks the installed tree against the SELECTED record; `test/harness-channel.test.ts` checks the declarations and holds each committed lockfile (`pnpm-lock.yaml` for alpha, `channels/rc/pnpm-lock.yaml` for rc) to its record, including that `@deepseek-ai/schemastery` resolves to one copy inside the cluster's range and that each family member the cluster peers on is the floor of the declared range. - `test/support/host.ts` — the real host, in-process, in two tiers: **A** (`mountTierA`: a bare cordis `Context`, `LlmRuntime` and `dsh-llm-pi-ai` mounted with `{ providers }`) for the `internal/config` waterfall and pi-ai's credential paths, and **B** (`bootTierB`: dsh-app-boot's `boot()` with a temporary HOME and a test bundle) for the config editor, `settings.describe`, the volatile commit and the characterization of the plugin as it ships. Its fakes (`fakeCredentials`, `fakeLaunchEnvironment`, `recordingFetch`) keep every case offline and its credentials out of the environment (a boot blanks the two keys the plugin reads directly, should the shell export them, and restores them on dispose); `S0_PROFILE` is the contract fixture, `CHARACTERIZED_PROFILE` (S0 plus a declared catalog-unknown id) the characterization's, and `S0_TODAY_BY_CELL` the measured per-model rows, tagged per facet, one table per host cell (`newest`, either channel's default install, and `rc-floor`/`alpha-floor`, the pis the desktop app ships; since the route claim every cell takes its channel's table (`s0PluginFor`), and the two channels' tables differ only in pi-ai's own defaults for the catalog-unknown id (`PI_AI_DEFAULTS`); `S0_TODAY` is the installed cell's, and `forHost`/`forChannel` pick any other per-cell or per-channel expectation outside the table, such as `PI_GAPS`), that the binary e2e shares (it reads its cell's `rows` directly; `todayRowsFor`, which only the characterization calls, answers the same rows for either apply order since the route claim). `logLines` captures every logger line at every level, because cordis's own buffer stops at `info` and tier B prints nothing to the console. Tier A has no loader, so the plugin's default host-module source cannot find pi-ai there; the harness swaps in `hostModulesForTests` through the seam in `src/cordis-context.ts`, and the route claim serves the planes in tier A as it does on the real host; `test/host-harness.test.ts` exercises the builder options no contract or characterization case passes, and tier B's two failure paths (an invalid first row, an exported key). Tier B's rows carry PACKAGE names through a stub module loader, never `cordis:` builtins, because the plugin's default host-module source finds pi-ai's entry by `options.name` (`piAiEntry` in `src/host/host-modules.ts`) and a builtin-named row would leave the route claim without pi-ai's module. It also fixes which of pi-ai and the plugin applies first (`order`), because the loader starts rows in parallel and the published binary applies the plugin first; the held row is released when the first one applies or its fiber settles, so an invalid first row fails instead of hanging the boot. -- `test/support/keyless-matrix.ts` — the keyless 401-vs-404 probe matrix's cell table and classifier. Pure, though it lives in `test/support/` rather than `src/utils/`: a function of its arguments, with no code path that can set a credential header; `test/e2e/keyless-matrix.e2e.ts` sends the cells. +- `test/support/keyless-matrix.ts` — the keyless 401-vs-404 probe matrix's cell table and classifier over three URL shapes (legacy `/zen(/go)/v1`, the root `/inference(/go)/v1`, modern `/inference(/go)/{openai,anthropic,google}/…`), the keyless free-model probe's cells and classifier (a `429 FreeUsageLimitError` is its own verdict, never a rate limit or a retry), the System One family's cells (recorded only), and the cells each `endpoint` setting dispatches to (`TODAY_DISPATCH`, `LEGACY_DISPATCH`). Pure, though it lives in `test/support/` rather than `src/utils/`: a function of its arguments, with no code path that can set a credential header; `test/e2e/keyless-matrix.e2e.ts` sends the cells. +- `test/support/keyed-probes.ts` — the keyed cells (the keyless ones with the plane's key in the header its protocol reads, and a body of a few tokens), their classifier, the recorded variants of the default completions cell (`keyedVariantCells`: `x-opencode-request`, `x-opencode-project` and `reasoning_effort: "none"`) with every 429's `Retry-After` (`retryAfterOf`), and the `OPENCODE_E2E_KEYED` tier every keyed live suite reads its key through (`armedKey`). Pure: it takes the variable's value and the key as arguments; `test/e2e/keyed-probes.e2e.ts` sends the cells. +- `test/support/endpoint-record.ts` — the dated markdown tables and lines both live probe suites write, which CI publishes and `docs/redirect-planes.md` §13 holds; `test/endpoint-record.test.ts` keeps that section in the formatter's columns and row order. - `test/support/live-walk.ts` · `test/support/live-probe.ts` — the live adapter blocks' candidate walk (pure: a protocol walks its candidates until a turn reaches the wire, and one no candidate reached is named, never dropped) and the raw protocol-shaped probe that confirms a withdrawn model before a walk moves past it (its tables are the ones `protocol-routing.e2e.ts` probes with). - `test/e2e/host/` — the measurement bundle (`probe-plugin.mjs`, its manifest and patch) the `e2e-host` job links into a throwaway profile under a temporary HOME, through a relative `link:` path. `test/e2e/host-binary.e2e.ts` is its reader, gated on `DSH_HOST_E2E=1`; it installs no host itself and needs `DSH_HOST_PREFIX`, a prefix holding the installed one. - `scripts/report-host-drift.ts` — the nightly harness-drift report (CI's `host-drift` job, schedule and dispatch only): reads the registry the way the records are defined (`dsh` at its dist-tag, pi's range, floor and newest, and every cordis-family member at the floor of the range the tagged cluster declares) and prints one `::warning::` per compared field that moved, exiting 0 whatever it finds, because a channel moving is the harness's release, not a regression. Its rules are `scripts/host-drift.ts` (`hostDrift`, pure), held by `test/host-drift.test.ts` with known answers, a loopback registry and the job's evaluated condition. P-T3 turns the report into a proposed bump pull request. @@ -104,7 +106,7 @@ It is the same trick `scripts/check.ts` already uses for the client bundle, at f - `src/purge-service.ts` · `src/purge-contract.ts` — the card's "purge deprecated models" action, and the shape it answers with. It **rewrites the profile**, deleting the declared model entries the live catalog has retired, through `configEditor` (the same service the settings UI writes with). That is deliberate: a read-time filter would leave the declaration in place, and then the only way to explain a missing row is to find the filter. One source of truth, and it is the configuration. - `src/session-cost.ts` · `src/cordis-context.ts` · `src/debug.ts` — per-turn accounting, the typed host interfaces, and stream debug logging. - `src/host/patch-method.ts` — `patchMethod`, the one way to wrap a method on a host object. A dispose restores only while its wrapper is still the current value (otherwise it becomes a pass-through under whatever wrapped on top), and deletes the own property when the original was inherited rather than assigning it back; a second install of the same site wraps the unpatched method instead of the first install. Identity is read from own descriptors, never from a property read, because cordis's traceable proxy answers a fresh proxy for every method read (`docs/engineering-notes.md`). `rewrapOnAdaptersUpdated` reinstalls a site on `llm/adapters-updated`, for a site that must follow the registry; none uses it yet. Every instance patch in `src/` goes through this module — today two sites: model discovery (`models-discovery.ts`) and the `settings.describe` graft (`host/describe-graft.ts`) — and `test/reload.test.ts` runs apply A, apply B, dispose A, dispose B on each. Two tripwires hold the rule. `test/architecture.test.ts` is textual: anywhere `apply()` reaches but the helper, it fails on a `Reflect.set`/`defineProperty`/`deleteProperty`/`setPrototypeOf`, an `Object.assign`/`defineProperty`/`defineProperties`/`setPrototypeOf` onto `llm`, an adapter, `settings` or `globalThis` (bare or through a property path such as `registration?.adapter`; a prototype swap shadows a method without leaving an own property), a dotted or bracketed assignment (through `?.` or `!` too) onto the first three, and `globalThis.x =` (the global `fetch` patch is the one named exemption, until P6); a write through a local alias or a cast receiver escapes it. The `apply()`-twice reload case is behavioural: it requires the helper's record on every own function `apply()` leaves on the stand-in's `llm` and `settings`. -- `src/utils/protocols.ts` · `src/utils/model-metadata.ts` · `src/utils/plane-model.ts` · `src/utils/plane-profile.ts` — the plane's pure decisions, each held at 100% coverage: the SDK → protocol table and the per-protocol base tables (every value measured; see `docs/redirect-planes.md`), the reasoning efforts a model offers (D1), the pi model object one declared model becomes (protocol from the catalog, then the route's `api`, and the base from the endpoint family table, never from pi's own catalog, D16, D26; metadata from the user's row first, D17), and the profile one plane entry becomes, with pi-ai's own checks and messages for every malformed field, never throwing; the default window and output tokens are the ones pi-ai's own `Config` fills in, never a copy (D17). +- `src/utils/protocols.ts` · `src/utils/model-metadata.ts` · `src/utils/plane-model.ts` · `src/utils/plane-profile.ts` — the plane's pure decisions, each held at 100% coverage: the SDK → protocol table and the per-protocol base tables (every value measured; see `docs/redirect-planes.md`), the reasoning efforts a model offers (D1), the pi model object one declared model becomes (protocol from the catalog, then the route's `api`, and the base from the endpoint family table, never from pi's own catalog, D16, D26; metadata from the user's row first, D17; the family `auto` resolves to, `AUTO_ENDPOINT_FAMILY`, P-R2), and the profile one plane entry becomes, with pi-ai's own checks and messages for every malformed field, never throwing; the default window and output tokens are the ones pi-ai's own `Config` fills in, never a copy (D17). - `src/attachment-services.ts` — optional attachment and filesystem services for the plane adapter's image requests, captured through Cordis child scopes. - `src/opencode-paths.ts` — where an OpenCode installation keeps its files, ported from `~/dev/opencode`'s `global-roots.ts` (the directory) and `database-path.ts` (the filename). It takes `home`, `env` and `channel` as parameters and reads neither the environment nor the filesystem, which is what makes the Windows and Linux layouts testable from here. Two facts it encodes that are easy to get wrong: the directory is XDG on EVERY platform — macOS and Windows use `~/.local/share` too, and `os.homedir()` is the only platform-dependent call — and the FILENAME is not constant, being `$OPENCODE_DB`, else `opencode.db` on a stable channel, else `opencode-.db`. The database sits under `data` while the config sits under `config`, so one root cannot serve both. It is the foundation of the opt-in reader: `go-discovery.ts`'s `resolveOpencodeStoreKey` is what imports it, and nothing else does. - `src/opencode-store.ts` — reading the credentials an OpenCode installation already holds, through `node:sqlite`. Two rules shape it: nothing throws (a missing file, a foreign schema and `:memory:` all answer `undefined`), and a value never leaves the return value — no log line, no error message, no client bundle. `node:sqlite` is host-only, which is what keeps the module on the host side of the bundle rule. It is reached only through `resolveOpencodeStoreKey`, which is the last step of every key-source policy and is off unless `readOpencodeStore` is on. Its second reader, `inspectOpencodeStore`, answers the _other_ question — "is there a credential here, and whose is it?" — with a sanitized summary (email, org, kind, expiry) and never the token, which is what lets the settings card show the evidence beside the permission. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 321fffa..a84fc3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,17 +36,20 @@ OPENCODE_E2E=1 OPENCODE_API_KEY=… OPENCODE_GO_API_KEY=… pnpm run test:e2e ``` - `OPENCODE_E2E=1` is required; without it every case skips, so a bare `pnpm run test:e2e` is a safe no-op. -- `OPENCODE_API_KEY` (Zen) arms the live `/models` enrichment case and the **paid** protocol-routing cases; `OPENCODE_GO_API_KEY` (Go plan) arms the live `/usage` case. Each block skips when its key is absent, which is why the CI `e2e` job stays green on fork PRs (secrets are not exposed to them) while still checking the endpoints answer. +- `OPENCODE_API_KEY` (Zen) arms the live `/models` enrichment case and the **paid** protocol-routing cases; `OPENCODE_GO_API_KEY` (Go plan) arms the live `/usage` case. Both keys also arm the keyed probes (`keyed-probes.e2e.ts`, each plane with its own key) on the `probes` tier and above, while the full keyed suites need the `full` tier (`OPENCODE_E2E_KEYED`, below). Each block skips when its key is absent, which is why the CI `e2e` job stays green on fork PRs (secrets are not exposed to them) while still checking the endpoints answer. - The **free-tier and unmetered** routing cases run keyless on purpose: measured 2026-10-06, a paid model answers `401 AuthError` on every endpoint without a credential, so keyless probing cannot tell the right PROTOCOL endpoint from a wrong one. Keyless protocol discrimination is real only for the free classes. What a paid model's keyless answer does still tell is whether a PATH exists at all — a removed path answers 404 — which is the one thing the keyless matrix below reads. -- `OPENCODE_ZEN_BASE_URL` / `OPENCODE_GO_BASE_URL` retarget `opencode-live` and `protocol-routing` at a mirror. The keyless matrix measures the vendor's own paths, which a mirror's base cannot name, so it skips while either is set. +- `OPENCODE_E2E_KEYED` is the tier of what may spend a key: `none` runs only the keyless cases, `probes` adds the minimal keyed probes (`keyed-probes.e2e.ts`), and `full` adds the full keyed suites (`protocol-routing`, `opencode-live`'s keyed blocks, `dispatch-levels`' live block). Unset means `full`, so a local run with keys arms everything; any other value fails the run. CI sets it per event — `probes` on a pull request that touches `src/host/**`, `src/utils/protocols.ts` (the legacy URLs the probes send and the protocol each probe model speaks), `src/utils/plane-model.ts` (the family the plugin dispatches to, `AUTO_ENDPOINT_FAMILY`, which the probes must keep matching), `src/identity.ts` (the CLI version in the user agent the probes carry), `src/utils/request-shape.ts` (the `anthropic-version` every Messages probe sends) or the probes' own inputs; `full` nightly and on dispatch; `none` otherwise — and passes the secrets only when it is not `none`. Nothing else arms them. Left out, and covered by the offline suite and the nightly run, which arms everything: the generated catalog (`src/catalog-data.ts`) and the live model choice (`src/models-catalog.ts`), which pick the probes' modern URLs and models and whose paths the keyless matrix checks on every pull request; and the `OPENCODE_UA` alias in `src/session.ts`, which a turn and a probe share, so `test/keyed-probes.test.ts` holds the probe's identity to the real fetch patch offline. +- `OPENCODE_E2E_RECORD_DIR`, when set, is where the keyless matrix and the keyed probes write their dated tables (`keyless.md`, `keyed.md`); CI publishes them to the run summary and the `endpoint-families` artifact, and a run's tables are pasted into `docs/redirect-planes.md` §13. +- `OPENCODE_ZEN_BASE_URL` / `OPENCODE_GO_BASE_URL` retarget `opencode-live` and `protocol-routing` at a mirror. The keyless matrix and the keyed probes measure the vendor's own paths, which a mirror's base cannot name, so both skip while either is set. The e2e files, one job each: - `opencode-live.e2e.ts` — the only place the plugin meets the real API. It catches what a stub cannot: **the vendor changing the payload**. The meter parses `/zen/go/v1/usage` and the picker consumes an enriched `/models` listing, so both shapes are asserted against the live service. The unkeyed cases assert reachability (never a 404) and that our header set does not change the gateway's verdict on a request — compared against the same call without it, rather than hardcoding a status the vendor may tighten. - `patched-fetch-headers.e2e.ts` — a local `node:http` listener, so the _outgoing_ header set crosses a real socket. The live gateway cannot echo a request back; everywhere else in the suite the injected headers are asserted against a captured `fetch`. -- `keyless-matrix.e2e.ts` — the keyless 401-vs-404 matrix: two planes × four protocols × the legacy (`/zen(/go)/v1`) and modern (`/inference(/go)/…`) endpoint families, one PAID model per cell from the live catalog — never a free one, which the gateway serves keyless with a real completion. It **sends no key and reads none**, so the `e2e` job runs it on every pull request, and it retries a cell once on a 429, a 5xx or a transport error before reading it. A 404 means the path is gone, a 401/403 that it exists, a 410 or a 400 "Model is unavailable" that the model is. It asserts that all sixteen cells have a model and were sent, that every answer is readable, and that no cell the plugin dispatches to today with the default endpoint (`auto`, an alias of `modern`; `TODAY_DISPATCH`: all four protocols on the modern family, per plane, since P-R1 moved completions there) is gone. The legacy verdicts are recorded for P-R2 — among them every cell an `endpoint: legacy` user dispatches to (completions, Responses and Messages on both planes, and Zen's Google), which are recorded, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell `TODAY_DISPATCH` already asserts. The cell table, the dispatch set and the classifier are a pure module (`test/support/keyless-matrix.ts`), unit-tested offline in `test/keyless-matrix.test.ts`, which also derives `TODAY_DISPATCH` from the URLs each host cell's rows measured (every cell reaches the same cells: the plugin builds every base from its own endpoint table). -- `protocol-routing.e2e.ts` — asks the gateway which endpoint serves each model's wire format, at the URL the plane adapter sends it to (the modern family by default, Go's Google included): keyless for the free classes, keyed (and skipped without a secret) for paid ones. Keyed probes of the legacy family per protocol are P-R2's record. Its last block sends paid models through the plane adapter on a tier-A host: each protocol walks up to three candidates (`walkPerProtocol`, `test/support/live-walk.ts`, unit-tested in `test/live-walk.test.ts`), moving past an entitlement answer or a withdrawal that a raw probe of the same id confirms (`test/support/live-probe.ts`), and a protocol no candidate reached skips the case by name rather than passing. -- `dispatch-levels.e2e.ts` — every thinking level the picker offers is one pi's own rule accepts, swept over the whole shipped catalog with no network; then a keyed live block sends one toggle model per plane and protocol through the plane adapter on a tier-A host with the picker's "On" effort, walking up to three toggle candidates per protocol the same way, and skips without the plane's key or, naming it, when a protocol had no entitled, still-served candidate. +- `keyless-matrix.e2e.ts` — the keyless 401-vs-404 matrix: two planes × four protocols × three URL shapes — legacy (`/zen(/go)/v1`), the root (`/inference(/go)/v1`, the provider-level base the catalog declared until 2026-10-16) and modern (`/inference(/go)/{openai,anthropic,google}/…`) — one PAID model per cell from the live catalog, never a free one, which the gateway serves keyless with a real completion. It **sends no key and reads none**, so the `e2e` job runs it on every pull request, and it retries a cell once on a 429, a 5xx or a transport error before reading it. A 404 means the path is gone, a 401/403 that it exists, a 410, a 400 "Model is unavailable" or a refusal of type `ModelDeprecated` at any status that the model is. It asserts that all twenty-four cells have a model and were sent, that every legacy and modern answer is readable (a root answer is recorded whatever it is: nothing dispatches there), and that no cell the plugin dispatches to today with the default endpoint (`auto`, an alias of `modern`; `TODAY_DISPATCH`: all four protocols on the modern family, per plane) is gone. A redirect is never followed (`redirect: "manual"`): it reads as `redirected`, unreadable. Beside the matrix, a keyless free-model probe sends a free completions model to `/inference/openai/v1/chat/completions` and asserts it is served or free-tier-gated; a retired free model (`model-gone`, a `ModelDeprecated` refusal included) or a busy upstream (a 503, a 529, or a 5xx naming an overload) moves on to the next candidate (`runFreeProbe`, which records the deciding candidate's own model and status), and a rate limit (a 429, or a 429 and then no answer), a busy upstream no candidate got past, or the free tier's usage cap (`429 FreeUsageLimitError`, its own verdict `free-usage-limited`, never retried and never read as served) skips it. The System One family's three shapes on the Zen plane (`/zen/v1/systemone`, `/inference/v1/systemone`, `/inference/systemone/v1/systemone`) are probed keyless with no model and recorded, never asserted: the plugin serves no classifier. `endpoint: legacy` is recorded, not guarded: the cells it dispatches to (`LEGACY_DISPATCH`: completions, Responses and Messages on both planes, and Zen's Google; Go's Google stays on `/inference/go/google/v1beta`, `GO_GOOGLE_BASE_URL`) must answer readably, like every legacy cell, but their path-gone is recorded and asserted nowhere live (Go's Google, whose cell under `endpoint: legacy` is the modern one, is guarded as a default cell), and `test/endpoint-family.test.ts` pins their mapping offline. Both probe suites rewrite their record after every cell, so a job killed at its time limit keeps what it measured. The cell table, the sets and the classifiers are a pure module (`test/support/keyless-matrix.ts`), unit-tested offline in `test/keyless-matrix.test.ts`, which also derives `TODAY_DISPATCH` from the URLs each host cell's rows measured (every cell reaches the same cells: the plugin builds every base from its own endpoint table). +- `keyed-probes.e2e.ts` — the keyed half: the same twenty-four cells, each sent with the plane's own key in the header its protocol reads, the plugin's identity headers (`PROBE_IDENTITY`: the turn's user agent, `x-opencode-client`, `x-opencode-project`, an in-band session id under both session headers — the Go plane refuses a keyed request without one; the per-request metric id `x-opencode-request` is left out on purpose) and a body of a few output tokens, on the `probes` and `full` tiers. Per plane it decides from the `TODAY_DISPATCH` cells (`planeOutcome`), each read beside the legacy and root cells of the same protocol, which carry the same model and key: it fails when one answers path-gone, wrong-protocol or auth-refused, a retired model another shape accepted, or nothing readable — `other`, a redirect, a refused session, `transport-error` or `no-model`. Only a wrong-protocol, path-gone, auth-refused or model-gone that the cell `endpoint: legacy` dispatches to — a different one, so never Go's Google — accepted is flip evidence; a refusal only the root or Go Google's `/zen/go` cell accepted is red but not evidence (no flip reaches it), a key refused on every shape that read it names the CI secret instead, and a protocol or a path the flip's cell measured nothing about asks for a re-run. The account's answer counts as acceptance beside a refused key or a gone path, never beside a refused protocol or model. It skips when none of them was served (the account's answer, a rate limit, a busy upstream, a retired model, a parsed but refused body), since that measured nothing. Every other cell is recorded. Like the keyless matrix, it skips while a mirror base URL is set. A second case per plane records the keyed variants of the default completions cell — a minted `x-opencode-request`, `x-opencode-project` as a folder name and as a 40-hex digest beside the identity's `global`, and a paid completions model that reasons with no `reasoning_effort` and with `"none"` — and every 429's `Retry-After`; they assert nothing (the plan's lessons L4, L5, L9 and L10, for P6 and P-R3). Headers and keys are never printed, and a body only through a redacting excerpt. The cells, tier, classifier and plane outcome are `test/support/keyed-probes.ts`, unit-tested in `test/keyed-probes.test.ts`, which also runs every keyed cell through the offline gateway mock's judge. +- `protocol-routing.e2e.ts` — asks the gateway which endpoint serves each model's wire format, at the URL the plane adapter sends it to (the modern family by default, Go's Google included): keyless for the free classes, keyed (and skipped without a secret, or below the `full` tier) for paid ones. The keyed record of every URL shape per protocol, the legacy family included, is `keyed-probes.e2e.ts`. Its last block sends paid models through the plane adapter on a tier-A host: each protocol walks up to three candidates (`walkPerProtocol`, `test/support/live-walk.ts`, unit-tested in `test/live-walk.test.ts`), moving past an entitlement answer or a withdrawal that a raw probe of the same id confirms (`test/support/live-probe.ts`), and a protocol no candidate reached skips the case by name rather than passing. +- `dispatch-levels.e2e.ts` — every thinking level the picker offers is one pi's own rule accepts, swept over the whole shipped catalog with no network; then a keyed live block, on the `full` tier, sends one toggle model per plane and protocol through the plane adapter on a tier-A host with the picker's "On" effort, walking up to three toggle candidates per protocol the same way, and skips without the plane's key, below the `full` tier, or, naming it, when a protocol had no entitled, still-served candidate. - `refusal-classification.e2e.ts` — the refusal classifier's verdicts on the gateway's bytes over a real `node:http` socket. - `host-binary.e2e.ts` — the published host binary; see [The real-host harness](#the-real-host-harness). It skips unless `DSH_HOST_E2E=1`. @@ -69,7 +72,7 @@ What every case does: - **Give tier-B cases 30 s**, and wait with a deadline (`waitForPlaneOwner`, 10 s by default, so a plane the plugin never claims fails by name inside the hook's budget), never a fixed sleep: the route claim registers the planes asynchronously, after pi-ai's commit. `piAiRestarts` counts pi-ai's restarts when a case needs to know how many the claim caused. - **A first row that fails does not hang the boot.** Tier B holds the second of pi-ai and the plugin until the first applies OR its fiber settles in any state, so a row whose config the schema rejects boots with that fiber FAILED (`fiber.await()` rethrows the real error) and the other row applied. -**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one channel's host, as one install runs it, lacks or breaks, its pi or its own code, that no phase of this plugin fixes: the install is a host cell for an in-process row, or an `e2e-host` cell, `npm` or `floor`, for the binary e2e; the title names the channel, the host and that install, and it is reported upstream, D26). `RowTag` in `test/support/host.ts` holds the one definition of the four. A phase that flips a row names it in its PR body; nothing else may move, except that P-R2's measured endpoint-family flip, if it happens, may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor` answers the cell's rows for either order), so the tags in `S0_TODAY` are what govern which entries a phase may flip. Since P-R1 the plugin places, describes and routes every declared id itself, so every cell takes its channel's table (`s0PluginFor`), and the two channels' tables differ only in pi-ai's own defaults for the catalog-unknown id (`PI_AI_DEFAULTS`); `forHost`/`forChannel` pick any other per-cell or per-channel expectation outside the table (such as `PI_GAPS`). P-T1b pinned the one facet that depended on the apply order (Go Gemini's call config, on the floors) with its own `[KNOWN_DEFECT]` case outside the booted blocks; P-R1 flipped that row, dropped `todayRowsFor`'s pi-ai-first override and turned the case into the `[BASELINE]` "no facet depends on the apply order" check, on every cell, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. +**Tags.** Every characterization row is titled `[BASELINE]` (must survive every phase), `[KNOWN_DEFECT]` (wrong today; a phase fixes it), `[INTENDED_CHANGE]` (right today; a phase changes it on purpose) or `[HOST_LIMITATION]` (something one channel's host, as one install runs it, lacks or breaks, its pi or its own code, that no phase of this plugin fixes: the install is a host cell for an in-process row, or an `e2e-host` cell, `npm` or `floor`, for the binary e2e; the title names the channel, the host and that install, and it is reported upstream, D26). `RowTag` in `test/support/host.ts` holds the one definition of the four. A phase that flips a row names it in its PR body; nothing else may move, except that the `fix:` that flips `AUTO_ENDPOINT_FAMILY` on P-R2's measured evidence (`docs/redirect-planes.md` §13) may move the family part of a `[BASELINE]` wire facet, re-tagging each one it moves and naming it in its PR body. The per-model rows live in `S0_TODAY_BY_CELL` in `host.ts` (`S0_TODAY` is the installed cell's), which the binary e2e reads too, and each model's three facets — model info, the call-config verdict, the wire — carry their own tag there: `host-characterization` generates one test per model and facet titled with that tag, in each apply order (`todayRowsFor` answers the cell's rows for either order), so the tags in `S0_TODAY` are what govern which entries a phase may flip. Since P-R1 the plugin places, describes and routes every declared id itself, so every cell takes its channel's table (`s0PluginFor`), and the two channels' tables differ only in pi-ai's own defaults for the catalog-unknown id (`PI_AI_DEFAULTS`); `forHost`/`forChannel` pick any other per-cell or per-channel expectation outside the table (such as `PI_GAPS`). P-T1b pinned the one facet that depended on the apply order (Go Gemini's call config, on the floors) with its own `[KNOWN_DEFECT]` case outside the booted blocks; P-R1 flipped that row, dropped `todayRowsFor`'s pi-ai-first override and turned the case into the `[BASELINE]` "no facet depends on the apply order" check, on every cell, in the same diff. The fixture is S0 plus one declared id no catalog knows (`CHARACTERIZED_PROFILE`); the contract tests keep plain S0. **A catalog refresh can touch the rows.** The rows' context windows and effort lists are copied from the generated `src/catalog-data.ts`, which the nightly refresh rewrites. A fixture-sanity case derives each row's context window and effort ids the way the plane adapter does (D17: the profile row's `contextWindow`, else the catalog; the efforts from `reasoningFor`), holds every cell's table to the same derivation, and names the model that moved; when it fails on a refresh pull request, update the row in `s0PluginFor` in `host.ts`, the table every cell shares. A phase that moves an `info` facet off that derivation on purpose adds that row's key to the case's `INFO_NOT_FROM_CATALOG` set in the same diff; the case refuses to exempt a `BASELINE` facet, and since P-R1 the set is empty. The `e2e-host` job's scope includes `test/support/host.ts`, so the binary e2e runs on that pull request too and checks the edited row against the published binary. diff --git a/docs/improvement-plan.md b/docs/improvement-plan.md index fa2784a..ce552d1 100644 --- a/docs/improvement-plan.md +++ b/docs/improvement-plan.md @@ -28,7 +28,8 @@ Phases land as **stacked pull requests**. Each one is based on the previous phas | P-T1b | #110 | `test:` a harness that is the real host | in review | **merged P-T1a**; re-measured on both channels, newest and floor: one row table per host cell (either channel's newest pi, and each floor), the binary e2e per channel (the `npm` cell on pull requests, the `floor` cell nightly), no pi patch anywhere | | P-H | #112 | `fix:` disabling or reloading the plugin restores the host methods it wrapped | in review | **merged P-T1b**; gains the Chinese "Changed in 1.x" bullet beside the English one (the pairing gate); the helper, its sites and its tripwires are unchanged; retitled from `refactor:`, because the restore on a disable or a reload is a defect fix the README announces and the changelog must carry (a `refactor` is hidden from it) | | P-R1 | #113 | `feat:` the plugin owns the plane routes | in review | **merged P-H**; on both channels' cells, one row table every cell shares (the plugin places, describes and routes every declared id itself); deletes the Gemini level wrapper (pi's own rule, from pi 1.3; the rc floor's pi 1.1.0 is a `HOST_LIMITATION`), reads no base or protocol from pi's catalog, and sets no default window or max tokens, pi-ai applies its own (D16, D17, D26) | -| P-R2 | #114 | `fix:` the endpoint family, measured | in review | pending: merges P-R1; gains the L3-L5, L9, L10, L13 probes | +| P-R2 | #114 | `test:` the endpoint families, measured | in review | **merged P-R1**; its dispatch set holds on every host cell; gains the L3, L4, L5, L9, L10 and L13 records and the per-request form of `main`'s mock-coverage gate | +| P-R2b | | `fix:` flip the default on §13's evidence | if needed | P-R2's first CI run decides it | | P3 | #115 | `feat:` Settings complements pi's list | in review | pending: merges P-R2 | | P-T2 | #116 | `test:` suite structure | in review | pending: merges P3; absorbs `main`'s tests and `docs/testing.md` | | P-T3 | | `ci:` CI and scripts | planned | shrinks: `main` extracted the scripts' pure halves; gains the bump pull request proposed from P-T1a's nightly host-drift report | @@ -325,13 +326,13 @@ What stays private is pinned by contract tests against the real host packages: **How each phase opens:** -- **Behaviour phases open with a red test:** P1a, P1b's fixes, P2, P-R1, P-R2, P3, P6 and P10b. -- **Pinning phases open with a green pin**, which the PR shows unchanged: P-T1a/b, P-H, P5, P7, P8, P9, P10a and P-T2. P-H's exceptions are red on the base by design: its reload test (the defect the phase removes) and the five cases in `test/models-discovery.test.ts` marked "a deliberate change in P-H" (a refused listing define, a reinstall's `registeredRoutes`, a frozen adapter, a refused adapter method, and an inherited adapter method left with no own property after dispose). +- **Behaviour phases open with a red test:** P1a, P1b's fixes, P2, P-R1, P3, P6 and P10b. +- **Pinning phases open with a green pin**, which the PR shows unchanged: P-T1a/b, P-H, P-R2, P5, P7, P8, P9, P10a and P-T2. P-H's exceptions are red on the base by design: its reload test (the defect the phase removes) and the five cases in `test/models-discovery.test.ts` marked "a deliberate change in P-H" (a refused listing define, a reinstall's `registeredRoutes`, a frozen adapter, a refused adapter method, and an inherited adapter method left with no own property after dispose). P-R2 landed its pins with the change in one commit: the default's and legacy's bases (the first and third describes of `test/endpoint-family.test.ts`, written against `baseUrlFor`) were run on the base (0cceb81) with the imports of the names the phase adds removed, and passed, with a control that changes one legacy base failing there; the constant and the dispatch-set cases land with the names they test (`AUTO_ENDPOINT_FAMILY`, `LEGACY_DISPATCH`, `MATRIX_FAMILIES`), and a measured flip of the default is P-R2b. - **P-T3 and P12 are exempt.** **Characterization rows are tagged when they are recorded**, so a later phase flips named rows. The authoritative per-facet tables are `S0_TODAY_BY_CELL` in `test/support/host.ts`, one per host cell (`S0_TODAY` is the installed cell's); each model's info, call-config verdict and wire carry their own tag, and a phase's PR body cites the rows it flips from there, per cell. The four tags: -- `BASELINE`: right today and must survive every phase. No phase may move a `BASELINE` facet, except that P-R2's measured family flip, if it happens, may move the endpoint-family part of a wire facet; it re-tags each one it moves and names it in its PR body. +- `BASELINE`: right today and must survive every phase. No phase may move a `BASELINE` facet, except that P-R2b, the `fix:` that flips `AUTO_ENDPOINT_FAMILY` on P-R2's measured evidence (`docs/redirect-planes.md` §13) may move the endpoint-family part of a wire facet; it re-tags each one it moves and names it in its PR body. - `KNOWN_DEFECT`, wrong today and a phase fixes it, for example: - Go Gemini answers `INVALID_CONFIG` — on the floor cells (rc's pi `1.1.0`, alpha's `1.3.0`, the desktop app's pi), in the published host's apply order (`plugin-first`); with pi-ai applying first it validates, so the row is order-dependent there (`todayRowsFor`). On `newest` (both channels' npm install, pi `1.5.0`) pi places the id and it validates in either order. P-R1 flips it and, in the same diff, drops `todayRowsFor`'s pi-ai-first override and turns the `[KNOWN_DEFECT]` apply-order case into a `[BASELINE]` "no facet depends on the apply order" check; - an unknown id is rejected; @@ -403,9 +404,18 @@ What stays private is pinned by contract tests against the real host packages: 7. **P-R1 `feat:` the plugin owns the plane routes.** In review. Landed as the design above (D1, D3, D5, D8, D13, D15-D21): `src/host/route-claim.ts`, `src/host/plane-provider.ts`, `src/host/describe-graft.ts` and `src/host/host-modules.ts` over the pure `src/utils/protocols.ts`, `model-metadata.ts`, `plane-model.ts` and `plane-profile.ts`, with `inject: []` at the root. Deleted: `responses-provider.ts`, `responses-routes.ts`, `google-bridge.ts`, `pi-provider.ts`, `utils/google-replay.ts` (a mock-gateway replay showed one provider label), the listing hide, the stream redirect, the model-info and plane-adapter decorations, and the D19 settings. Nine deliberate refinements of the text above: the describe wrapper is its own module on `patchMethod`; the host-module seam sits in `cordis-context.ts` with the loaders in `host/host-modules.ts`, and pi-ai's module is taken from its own loader entry during its commit run; the libraries (`loadHostPi`'s pi, dsh-llm, the loader) are found from the wrapper the loader resolves from the context's base URL, never from the plugin's own install, so a `link:`ed checkout runs on the host's copies too (the binary e2e asserts the pi root is the host prefix's); D20 mirrors pi-ai's exact rule (the launch environment only when no credentials service is mounted); `legacy` took pi's own base when pi filed the model under the same protocol on a `/zen` root (withdrawn in the restack, D26: the family table is the only base source); cost is catalog > pi > zero, because pi-ai's model row has no `cost` (D17's table puts it under the user's row), and nothing replaces `modelRoutes`' per-model protocol pin — the route-level `api` covers only ids the catalog does not classify (D26); only one pi-ai row's planes are stripped and served — the served row is sticky while it is live and declares a plane, and otherwise the first live row whose raw config declares one, else the first live row, so a re-enabled row that cordis appends to the end of the registry is still the one served, and the served row never moves between two live declaring rows, which would leave the one left behind stripped with nobody serving its planes — and another pi-ai row keeps and serves its own, reported once; when the served row changes (it was disposed or stopped declaring), what was built from the previous one is released and a new served row that already holds a plane is restarted once, as the mount does; the teardown and the stand-down wait for a LOADING pi-ai to settle and restart it if it resolved without a plane it declares (the text said "with pi-ai ACTIVE"), and the stand-down releases the plugin's handle first and happens whether or not a plane is declared, so a plane a later Settings save adds goes to pi-ai's own save check rather than to a refusal no retry can clear; and the characterization's fixture-sanity case derives `info` as D17 does, so `INFO_NOT_FROM_CATALOG` stays empty. The profiles' memo key holds the catalog's generation and a landed refresh re-runs the reconcile, so the live catalog reaches the profiles after boot without a plane edit. The PR body names every tagged row it flips. - **From the P-R1 review.** (a) **D1 for an effort list that names `none`:** on Chat Completions the `none` is Off's WIRE value (`thinkingLevelMap.off = "none"`, profile `reasoningEfforts.off = "none"`), as pi 1.1.0's own entries carry it, so Zen `mistral-large-4` and Go `hy3`/`hy4-preview` keep offering Off and send `reasoning_effort: "none"` for Off and for a turn with no level — what pi-ai served before P-R1. On Responses a listed `none` still offers no Off: pi's own entries disagree there (gpt-5.x pin `off: null`, gpt-6 carry `"none"`), a string `off` would become the request every no-level turn sends, and `opencode/gpt-5.4`'s `efforts` facet is `[BASELINE]` without Off, so it is the owner's call. **Open for the owner, not landed:** a `toggle` beside an `effort` list (Zen/Go `claude-haiku-5-5`, `deepseek-v4*`, `kimi-k3`, `qwen3.8-*`) offers the list's levels without Off, as the plugin did before P-R1 (`[BASELINE]`); pi alone offers Off on `claude-haiku-5-5`. Extending D1 to offer Off there moves `[BASELINE]` facets and changes the no-level request on Anthropic and Responses, so it needs a decision and its own characterization flips. (b) **D20 on the Go plane:** pi's lookup reads `OPENCODE_API_KEY` on both planes, so a Go entry without `apiKeyEnv` sends the Zen key on every protocol; before P-R1 Go's Responses, Messages and Google defaulted to `OPENCODE_GO_API_KEY`. D20 stands as written; the README's "Changed in 1.x" names the change and step 3 tells a Go user to name the reference, and `test/host-planes.test.ts` pins both cases. Changing the Go default would amend D20. (c) **A Settings save is judged with the last modules that loaded**, so a valid save in the window after a handover or pi-ai's partial dispose is not refused for timing; only a save before the mount's first load is. (d) **The Models store's join is a restatement**, like `hasProviderApiKey`: `modelsPageRows` in `test/support/host.ts` and the probe's `modelsJoin` restate `configured` and `apiKeyEnvOf` from `client/ui-settings-models/src/client/store.ts` (`joinProviderDirectory` and `read`, at the lines `modelsPageRows`' own doc cites, read at each channel record's tag), over the real host's `settings.describe({ redactSecrets: true })` and `listConfigurableProviders()` in tier B and on the binary. The client package is not added as a dependency: it would pull a client UI package into a node test for a three-line join. - **Restacked onto both channels (D14'', D22'', D26).** Merged P-H, so it runs on either channel's newest pi and each floor. One row table serves every host cell, since the plugin places, describes and routes every declared id itself; a row differs by cell only where the host decides (pi-ai's own defaults for the catalog-unknown id, per channel). Three removals of compensation for the host's pi: the Gemini level wrapper (`src/host/google-levels.ts`, its wiring, test, mock case, coverage floor and docs) goes, so the Google thinking control is pi's own; pi's catalog entry leaves the protocol order (catalog SDK, then the route's `api`, then completions) and the base (the family table only), so a pi that reshapes its catalog moves no turn; and `plane-profile.ts` restates none of pi-ai's default window and output tokens, reading back what pi-ai's `Config` filled. What the wrapper's removal exposes is recorded, not fixed: pi `1.1.0` (the `rc` floor, its desktop app's pi) sends the level-only Gemini 4 model a thinking budget on either plane (measured live on Go), a `[HOST_LIMITATION]` row in the in-process characterization and the binary e2e's floor cell, with the mock and live cases that need the level skipping by pi's own `usesGoogleThinkingLevel`, the READMEs' "Host limitations" list, and an upstream report. Measured on the published binaries: both cells of `rc` and the `alpha` floor serve the planes through the claim; on `alpha`'s npm cell cordis `4.0.6-alpha.2` hands `internal/config` listeners the fiber and reads the config from `next()`, a contract the claim does not speak, so pi-ai serves the planes there — a `[KNOWN_DEFECT]` row (detected from the contract the probe saw dispatched), which the alpha bump to `0.2.2-alpha.3` (requiring that cordis) must fix before it lands. -8. **P-R2 `fix:` the endpoint family, measured.** - - Probe 3 URL shapes × 4 protocols × 2 planes, keyless, plus keyed probes per family with the CI keys. - - Set the default family and pin it with a test. If a `modern` cell fails, flip the default; `endpoint: legacy` still maps every protocol except Go's Google, which stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`). Decide here whether `endpoint: legacy` is a guarded configuration: the keyless matrix records its Responses and Messages cells and Zen's Google cell but does not assert them. +8. **P-R2 `test:` the endpoint families, measured.** In review. Landed as: + - **The default in one constant.** `AUTO_ENDPOINT_FAMILY` (`src/utils/plane-model.ts`) is the family `auto` resolves to, `modern`, and `test/endpoint-family.test.ts` pins it, the default's base for every plane × protocol, `endpoint: legacy`'s mapping of every protocol on both planes (Go's Google on `GO_GOOGLE_BASE_URL`), and the matrix cells a turn reaches under each setting. A naming refactor: no behaviour changes and no characterization row flips. + - **Keyless:** 3 URL shapes (legacy `/zen(/go)/v1`, the root `/inference(/go)/v1` — the provider-level base the catalog declared until 2026-10-16, which now declares the modern completions base — modern `/inference(/go)/{openai,anthropic,google}/…`) × 4 protocols × 2 planes, 24 cells, plus one keyless free-model probe on `/inference/openai/v1/chat/completions`. The root shape is recorded only: nothing dispatches there. + - **Keyed:** one minimal probe per plane × protocol × shape with the plane's own key and the identity headers the fetch patch puts on a turn, its user agent included (`test/e2e/keyed-probes.e2e.ts`), deciding each plane from the cells the default dispatches to, each read beside the legacy and root cells of the same plane × protocol (`planeOutcome`). + - **Pure classifiers and decisions** for each kind of answer and each pass, skip or fail (`test/support/keyless-matrix.ts`, `test/support/keyed-probes.ts`: the verdict of a retried probe, the unreadable and gone rows, the free probe's outcome, a keyed plane's outcome), unit-tested offline; the keyed cells also go through the offline gateway mock's judge. + - **Tiers.** `OPENCODE_E2E_KEYED` is `none`, `probes` or `full` (unset is `full`); the full keyed suites read their keys through `armedKey(tier, "full", …)`. + - **The record.** Both live suites write a dated table (`test/support/endpoint-record.ts`), rewritten after every cell so a job killed at its time limit keeps what it measured, that CI publishes to the run summary and the `endpoint-families` artifact. No probe follows a redirect; a redirected cell records the path it pointed at. + - **Decided:** `endpoint: legacy` is recorded, not guarded, on the live gateway (`LEGACY_DISPATCH`, marked `†`): its keyless answers must be readable, like every legacy cell, but its path-gone is asserted nowhere live, except Go's Google, which is also a default cell and is guarded as one; its mapping is guarded by the unit pins. + - **The flip rule.** A keyed modern cell that answers path-gone, wrong-protocol or auth-refused with the CI key turns the job red, and so does a modern model-gone beside another shape of the same plane × protocol (same model, same key) that accepted it (`served` or `path-exists`). Flip evidence is a wrong-protocol, path-gone, auth-refused or model-gone on the modern cell while the cell a flip would send the same plane × protocol to (`LEGACY_DISPATCH`'s, when it is not the modern cell itself — never Go's Google) accepted the request (`served` or `path-exists`; beside a refused key or a gone path `account` counts too, as it shows the key read and the path there, but not beside a refused protocol or model, as the account's gate answers before either is read): the evidence for P-R2b, the `fix:` that flips `AUTO_ENDPOINT_FAMILY`, with the copy, `TODAY_DISPATCH` and the re-tagged wire facets. A refusal only the root or Go Google's `/zen/go` cell accepted is red but not flip evidence (no flip reaches either); so are a key refused on every shape that read it (the CI secret), a key no other shape read, and a protocol or a path the flip's cell refused too (a flip would not serve it). Each sibling counts only for what it measured: a protocol or a path the flip's cell answered nothing about (a rate limit, a busy upstream, the account's gate, no answer) is red, not evidence, and a re-run. A busy upstream never counts as a shape accepting the request. The job also fails when a default cell answers nothing readable (`other`, a redirect, a refused session, `transport-error`, `no-model`), which is not flip evidence, and skips a plane none of whose default cells was served (a busy upstream — a 503, a 529 or a 5xx naming an overload — skips like a rate limit; a 500 or 502 stays `other`). + - **Restacked onto 1.8.0 (the lessons from opencode2dsh).** Merged P-R1, so the dispatch set holds on every host cell (the plugin builds every base from its own endpoint table, D26), and `ModelDeprecated` reads as a withdrawn model in a turn's failure message and the live walk's raw probe (P-R1's `isModelGoneMessage` and `modelGone`); P-R2 extends the type to its own probe classifiers — keyless, free and keyed (`isModelDeprecated`) — so a refusal of that type at any status is `model-gone`, a retired free model moves the free probe on rather than failing it, and a keyed 404 or 400 of that type is never a gone path or a shape that accepted the model. Added, all recorded and none asserted: the free probe's `429 FreeUsageLimitError` is its own verdict, `free-usage-limited`, a skip that is never retried and never read as served (L3; the root family L3 asks for was already a shape); the System One family's three shapes on the Zen plane, keyless (L13); the keyed variants of each plane's default completions cell, a minted `x-opencode-request` (L4), `x-opencode-project` as `global`, a folder name and a 40-hex digest (L5), and `reasoning_effort: "none"` beside no effort on a paid completions model that reasons (L10); and every keyed 429's `Retry-After` (L9). `main`'s live-run verdicts (`test/e2e/verdict.ts`) read the usage cap as the free tier closing, not a busy gateway, and the keyed free-tier walk in `protocol-routing.e2e.ts` skips naming the cap when any candidate met it and none met a busy gateway (`freeWalkGap`: busy, then capped, then closed), since a capped candidate says nothing about the tier, rather than reporting a session gate as the reason; a session gate another candidate met is still named in the skip reason (`freeWalkCappedReason`). The combined Google User-Agent question closed offline rather than live: the fetch patch replaces the User-Agent on the request every protocol sends, Google's included, and the mock pins Google's wire value to `OPENCODE_UA` on both planes (`test/mock/protocol-compat.mock.ts`), so the gateway never sees a combined value (`docs/redirect-planes.md` §9); it reopens only if P10a moves the agent into call options. `main`'s mock-coverage gate (#142) gains its per-request form over `TODAY_DISPATCH ∪ LEGACY_DISPATCH`, with every root cell required to answer `no_route`. + - **Measured results** live in `docs/redirect-planes.md` §13, pending the first CI run. + - **The follow-up is P-R2b**, conditional: it pastes the first CI run's tables into §13, and flips `AUTO_ENDPOINT_FAMILY` only if that run (or a later one) shows flip evidence; with none, it closes as a `docs:` change once §13 holds a real run. 9. **P3 `feat:` Settings complements pi's list.** - The `discoverModels` wrapper answers servable, non-deprecated models, deduped and whitelisted. - Delete `installModelDiscovery` and `toDiscovered`. @@ -452,7 +462,7 @@ What stays private is pinned by contract tests against the real host packages: ## Order ``` -P0 → P1a → P1b → P2 → P-T1a → P-T1b → P-H → P-R1 → P-R2 → P3 → P-T2 → P-T3 → P-R3 → P5 → P6 → P7 → P8 → P9 → P10a → P10b → P11 → P12 +P0 → P1a → P1b → P2 → P-T1a → P-T1b → P-H → P-R1 → P-R2 → (P-R2b) → P3 → P-T2 → P-T3 → P-R3 → P5 → P6 → P7 → P8 → P9 → P10a → P10b → P11 → P12 ``` - **P1a first:** it fixes what already shipped broken. @@ -475,7 +485,7 @@ How each phase changes now that it sits on `main` 1.8.0 and on both harness chan | P-T1b | Runs tiers A and B on both channels (D14''), the floors nightly; rows carry one expectation per host cell (`newest`, shared by both channels, and each floor); pi-ai's own Config defaults (window and max tokens) are keyed by channel (`host-shapes`); the declared unknown id answers `INVALID_CONFIG` on every cell; contract tests run on both channels; contract-test citations move to cordis `4.0.6-alpha.1`. | | P-H | Gains the Chinese "Changed in 1.x" bullet the pairing gate requires. | | P-R1 | Implements D26: deletes `src/host/google-levels.ts` with its wiring, test, mock case, coverage floor and docs lines; sets no default window or max tokens (pi-ai applies its own, D17); takes no base or protocol from pi's catalog entries (D16). Deletes `test/google-thinking.test.ts` with `src/google-bridge.ts`, moving its intents to the mock suite; takes `main`'s model lists with its own routing prose. Owns what the deletion exposes (D26): the `HOST_LIMITATION` row for Gemini 4 thinking levels on `rc`'s floor pi `1.1.0`, the feature-detected skips (pi's own exported predicate) in that mock case and the live Gemini-level case, so the nightly `rc` floor cell stays green, the "Host limitations" list the README's support policy promises, and the upstream report. | -| P-R2 | Splits the free probe's 429: `FreeUsageLimitError` is its own non-retryable verdict, not a rate limit; `ModelDeprecated` reads as gone; adds the probe rows of lessons L3, L4, L5, L9, L10 and L13; ports `main`'s mock-coverage gate (#142) over the dispatch tables. | +| P-R2 | Splits the free probe's 429: `FreeUsageLimitError` is its own non-retryable verdict, not a rate limit; `ModelDeprecated` reads as gone in the probe classifiers (P-R1 brought it to the turn pre-filter); adds the probe rows of lessons L3, L4, L5, L9, L10 and L13; ports `main`'s mock-coverage gate (#142) over the dispatch tables, each dispatched cell required to reach its own protocol's endpoint. | | P3 | Unchanged seams on both channels' hosts; its Settings fetch row runs in each channel's binary install, newest and floor. | | P-T2 | Absorbs `main`'s tests and owns `docs/testing.md`; folds `test/e2e/verdict.ts` into the live-probe module; tests `main`'s plane lookup fallback. | | P-T3 | **Shrinks.** `main` already extracted the scripts' pure halves and wrote `docs/testing.md`. Remaining: one unit run with the build before the `lib/` readers, `test/**/*.tsx` type-checked, the composite setup action, `check.ts` trimmed to freshness, consumer install and size, `verify-*` exit codes, and `no-local-paths` over `.github/` and `*.sh`. **New:** the CLI-version script reads the registry's dist-tags and the 2.x User-Agent format (L6); the nightly host-drift report P-T1a added becomes a proposed bump pull request that moves CONTRIBUTING's release-step-3 data, runs the tiers and the binary e2e (dispatching the floor cells when the bump moves `piFloor` or a family field), and reports the rows that go red for the reviewed re-measure rather than editing them (D22''), and a cut opening a new prerelease tuple goes up for review as a range change. (L13's newest-pi cell is no longer P-T3's: under D14'' each channel's default install already is the newest pi.) | @@ -622,8 +632,15 @@ Two community plugins solve the same problem on the same host: [`FishBottle7/ope **P-R2** -- **Live e2e:** the keyless matrix, keyed probes per family, and one keyless free-model probe. -- **Unit:** the default family. +- **Unit:** + - the default family: `AUTO_ENDPOINT_FAMILY` is `modern`, `auto` follows it, and the default's base per plane × protocol; + - `endpoint: legacy` maps every protocol on both planes, and the cells each setting reaches (`TODAY_DISPATCH`, `LEGACY_DISPATCH`); + - the 24-cell table, the free probe's cells, every classifier, the tiers, the keyed cells against the plugin's own `authHeaderFor`, the record formatter, §13's shape and the CI wiring; + - the usage cap's own verdict and no retry, the System One cells, each keyed variant against the baseline it changes, the `Retry-After` reader, and the mock serving every dispatched request and no root one. +- **Live e2e:** + - keyless, every tier: the 24-cell matrix, the free-model probe and the System One record; + - `probes`: one keyed probe per plane × protocol × shape, and the recorded variants; + - `full`: the full keyed suites. **P3** @@ -666,9 +683,19 @@ The same cordis hands `internal/config` listeners the fiber and reads the config ### The live gateway job (`OPENCODE_API_KEY` / `OPENCODE_GO_API_KEY`) -- **Every PR:** the keyless matrix. A 404 means the path is gone; a 400 "Model is unavailable" or a 410 means the model is gone. It asserts that every cell has a model and was sent, and that no cell the plugin dispatches to with the default endpoint (`auto`, an alias of `modern`: since P-R1, all four protocols on the modern family, on every host cell) answers 404; the legacy verdicts are P-R2's record. `endpoint: legacy` users dispatch completions, Responses and Messages on both planes, and Zen's Google, to the legacy cells, which are recorded for P-R2, not asserted; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), the modern cell already asserted. -- **PRs that touch `src/host/**`, and nightly:** minimal keyed probes, one per plane × protocol, with a small `max_tokens`. -- **Nightly and on dispatch:** the full keyed suites. +What spends a key is a tier, `OPENCODE_E2E_KEYED`, which a step of the `e2e` job sets per event; the keys are passed only when it is not `none`: + +| Run | Tier | What runs | +| :-- | :-- | :-- | +| a PR | `none` | the keyless cases only, no key spent: the matrix, the free probe, the System One family's three shapes (recorded), and the keyless cases of protocol-routing and opencode-live | +| a PR touching `src/host/**`, `src/utils/protocols.ts`, `src/utils/plane-model.ts`, `src/identity.ts`, `src/utils/request-shape.ts` or the probes' own inputs | `probes` | adds the keyed probes: one request of a few tokens per plane × protocol × URL shape, plus six recorded variants of each plane's default completions cell (request id, project, reasoning effort) and every 429's `Retry-After` | +| nightly, dispatch | `full` | adds the full keyed suites | +| a push to main | `none` | keyless only | + +- **The keyless matrix** (every run): 3 shapes × 4 protocols × 2 planes. A 404 means the path is gone; a 400 "Model is unavailable", a 410 or a refusal of type `ModelDeprecated` at any status means the model is gone. It asserts that every cell has a model and was sent, that every legacy and modern answer is readable (a redirect, a rate limit or no answer is not), and that no cell the default dispatches to (`TODAY_DISPATCH`: all four protocols on the modern family, on every host cell) answers 404. The free probe asserts that a free completions model is served or free-tier-gated on `/inference/openai/v1/chat/completions`; a retired model (a `ModelDeprecated` refusal included) or a busy upstream moves on to the next candidate (`runFreeProbe`, which records the deciding candidate's own model and status), and it skips on a rate limit, a busy upstream no candidate got past, or the free tier's usage cap (`429 FreeUsageLimitError`, `free-usage-limited`), never retried and never read as served. +- **The keyed probes** (`probes` and `full`), per plane, from the `TODAY_DISPATCH` cells, each read beside the legacy and root cells of the same protocol (same model, same key): it fails when one answers path-gone, wrong-protocol or auth-refused, a retired model another shape accepted, or nothing readable — `other`, a redirect, a refused session, `transport-error` or `no-model`. Flip evidence is a wrong-protocol, path-gone, auth-refused or model-gone on the modern cell while the cell `endpoint: legacy` dispatches to — a different one — accepted the request (an unreadable answer never is); a refusal only the root or Go Google's `/zen/go` cell accepted is red but not evidence, a key refused on every shape that read it names the CI secret instead, and a protocol or a path the flip's cell measured nothing about asks for a re-run. It skips when none of them was served (the account's answer, a rate limit, a busy upstream, a retired model, a parsed but refused body), since that measured nothing. +- `endpoint: legacy`'s cells (completions, Responses and Messages on both planes, and Zen's Google) and the root shape are recorded, not asserted on their own — a legacy keyless answer must still be readable, but its path-gone fails nothing; Go's Google stays on `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL`), a default cell guarded as one. +- Both probe suites write a dated table to the run summary and the `endpoint-families` artifact; a run's tables are pasted into `docs/redirect-planes.md` §13. ## How each phase is built diff --git a/docs/redirect-planes.md b/docs/redirect-planes.md index e195321..f7a8bfe 100644 --- a/docs/redirect-planes.md +++ b/docs/redirect-planes.md @@ -15,7 +15,7 @@ A **plane** is one gateway base URL with its own catalog and its own credential. Counts as of 2026-11-03 (the shipped shim, regenerated in #132 and refreshed for `pulse-alpha-free`); the document's 2026-10-09 stamp is for its measurements. -**There is no longer one base URL per plane to quote.** The gateway answers on two endpoint families and the `endpoint` setting picks one: `auto`, the default, is an alias of `modern`, the per-protocol inference family (`…/inference/openai/v1` and `…/inference/go/openai/v1` for completions), and `legacy` pins the older `…/zen/v1` and `…/zen/go/v1`. Both are still served and they are entitled separately, which is why the choice is the user's rather than a migration. The dated measurement below was taken on the legacy family. +**There is no longer one base URL per plane to quote.** The gateway answers on two endpoint families and the `endpoint` setting picks one: `auto`, the default, is an alias of `modern` (`AUTO_ENDPOINT_FAMILY` in `src/utils/plane-model.ts`), the per-protocol `…/inference/{openai,anthropic,google}/…` and `…/inference/go/…`; `legacy` pins the older `…/zen/v1` and `…/zen/go/v1`, except Go's Google, which has no legacy base and stays on `…/inference/go/google/v1beta`. Which families answer, keyless and with each plane's own key, is measured in CI (§13), not assumed. The dated measurement below was taken on the legacy family. A plane is a property of the **account and the base URL**, never of a model. The same model id can exist on both planes and be served on a different shape in each. @@ -74,6 +74,9 @@ These are the rules a correct implementation satisfies. Each one has a test in | `protocol-routing.e2e.ts` · live probe | I1, I3 | a protocol is declared routable without a path and an auth convention that were really sent, on either plane | | `test/mock/*.mock.ts` · offline gateway | §9's shape table | a plane × protocol × family sends a path, query or wire header its endpoint would refuse, or the mock answers a request the vendor would refuse | | `test/mock/plane-dispatch.mock.ts` · the plane adapter | I1–I4 | a characterized model on either plane reaches a path, model or credential header other than its protocol's, through the real `LlmRuntime` | +| `keyless-matrix.e2e.ts` · live, keyless | I3 | a cell the default sends to answers 404, or a free model is not parsed on the default completions path | +| `keyed-probes.e2e.ts` · live, keyed | I1, I3 | a default-family cell answers path-gone, wrong-protocol or auth-refused with the plane's own key, a retired model another shape of the same protocol accepted, or nothing readable: `other`, a redirect, a refused session, `transport-error`, `no-model`. Flip evidence only when the cell `endpoint: legacy` dispatches to, a different one, accepted what the default cell refused (§13) | +| `endpoint-family.test.ts` | I3 | `auto` stops meaning the pinned family, or legacy stops mapping a protocol on a plane | | coverage ratchet | — | a new branch of the plane model or profile is added without a test (both are held at 100%) | ## 6. Refactor (done, then superseded) @@ -92,7 +95,7 @@ The gap was structural, not a missing branch: **the plane was not part of a rout - The §5 guards. The plane × SDK case was written first and **failed against the code as it was** - `opencode-go + @ai-sdk/openai` and `+ @ai-sdk/anthropic` - then passed once the Go routes landed, which is what makes it a check rather than a restatement. - `pnpm run test:mock` — the offline gateway mock (`test/mock/`): a Vite dev server that answers every endpoint in its own wire format and refuses a request whose shape its endpoint does not accept — an unknown path, a model sent to an endpoint that cannot speak its format, a missing session header, a placeholder credential. It drives the real path (pi's protocol implementation → `patchFetch` → the mock) across both planes, all four protocols and both endpoint families, so §9's table is exercised rather than quoted. No key and no network, so it gates every push. -- `pnpm run test:e2e` with keys present. The probe now takes the plane, and one case sends a real request to the Go base with the Go credential, asserting only that the gateway PARSED the model - any answer but 404 or 500 - so the case does not bet on an entitlement. +- `pnpm run test:e2e` with keys present. The probe now takes the plane, and one case sends a real request to the Go base with the Go credential, asserting only that the gateway PARSED the model - any answer but 404 or 500 - so the case does not bet on an entitlement. What spends a key is tiered by `OPENCODE_E2E_KEYED`: `none` spends no key, so only the keyless cases run (the keyless matrix, the free probe, and the keyless cases of protocol-routing and opencode-live), `probes` adds the minimal keyed probes (`keyed-probes.e2e.ts`), and `full` — the default when it is unset, and what CI runs nightly and on dispatch — adds the full keyed suites. CI runs `probes` on a pull request that touches `src/host/**`, `src/utils/protocols.ts` (the legacy URLs the probes send and the protocol each probe model speaks), `src/utils/plane-model.ts` (the family the plugin dispatches to, `AUTO_ENDPOINT_FAMILY`, which the probes must keep matching), `src/identity.ts` (the CLI version in the user agent the probes carry), `src/utils/request-shape.ts` (the `anthropic-version` every Messages probe sends) or the probes' own inputs, `full` nightly and on dispatch, and `none` otherwise. Nothing else arms them. Left out, and covered by the offline suite and the nightly run, which arms everything: the generated catalog (`src/catalog-data.ts`) and the live model choice (`src/models-catalog.ts`), which pick the probes' modern URLs and models and whose paths the keyless matrix checks on every pull request; and the `OPENCODE_UA` alias in `src/session.ts`, which a turn and a probe share, so `test/keyed-probes.test.ts` holds the probe's identity to the real fetch patch offline. - The Zen plane compared before and after: route ids, mounted model ids and base URLs must be byte-identical. ## 8. The full path, front to back @@ -143,7 +146,7 @@ Measured 2026-10-10 by building one with pi's own protocol implementations and i **The session header reaches every protocol, and the rest of the gateway headers do not come from the provider.** `buildPlaneProvider` wraps each `api` with pi's own `withOpenCodeSessionHeader` before `createProvider`, and that is all it adds per call: pi's wrapper sets `x-opencode-session` from the turn's session id on all four protocols, and steps aside when the request already carries one. `x-opencode-client`, `x-opencode-project`, the derived `ses_` id and the User-Agent are set by the global fetch patch on the request the protocol sends, not on the call's options; moving them into each call (`options.headers`) is P10a's first move. -**One detail left open:** the Google implementation sets its own `User-Agent` inside the protocol, so ours arrives appended rather than replacing it (`pi (darwin …), opencode/1.19.5 …`). Whether the gateway accepts the combined value or wants a single agent string is unverified. +**The User-Agent is one value on every protocol, Google's included.** The Google implementation sets an agent string of its own inside the protocol, but the fetch patch sets `User-Agent` on the request that protocol sends, which replaces whatever pi and the Google SDK put there rather than appending to it. The offline mock pins the wire value to `OPENCODE_UA` for the Google protocol on both planes (`test/mock/protocol-compat.mock.ts`), so the gateway never sees a combined value, and §13's keyed Google cells measure the agent string a turn actually sends. The question reopens only if P10a moves the agent into each call's options instead of the fetch patch. ## 10. A catalog route serves every protocol already @@ -192,3 +195,84 @@ What carried over from the bridge, and why: - **A Gemini model's thinking control is pi's own (D26).** Google has two thinking controls and a model accepts only its own: a level model refuses a token `thinkingBudget` with 400 INVALID_ARGUMENT ("Thinking budget is not supported for this model. Use thinking level instead.", measured live on Go `gemini-4-argon`). pi decides which control a request carries by its own rule (`usesGoogleThinkingLevel`), which names Gemini 4 from pi `1.3.0` on; the plugin sends what the host's pi builds and re-derives nothing. pi `1.1.0`, the `rc` channel's floor and its desktop app's pi, does not name it, so every `gemini-4-argon` turn there carries a thinking budget on either plane, a token budget for a chosen effort and `thinkingBudget: 0` (pi's disable) when none is chosen, which that model cannot honour since it cannot turn thinking off (the rule reads the model id, not the plane; the refusal is measured live on Go, and Zen's, expected under the same rule, is not yet measured): a `[HOST_LIMITATION]` row (`test/host-characterization.test.ts`, and the binary e2e's floor cell), reported upstream, with the mock and live cases that need the level skipping by pi's own predicate. A profile that declares Off for a level model gets pi's own disable, `thinkingBudget: 0`; a model that cannot turn thinking off (`gemini-4-argon`) refuses it, so declaring Off there is a profile error. What is verified, and in what composition: the in-process characterization (`test/host-characterization.test.ts`) and the real-host suite (`test/host-planes.test.ts`) run the plugin on the real `llm` service and `llm-pi-ai` in both apply orders, and the binary e2e (`test/e2e/host-binary.e2e.ts`) runs the built bundle on a published host, asserting, in every cell where the route claim engages, that both planes are the plugin's `PlaneAdapter` and that a Go Gemini turn is sent to `https://opencode.ai/inference/go/google/v1beta`, with a `thinkingLevel` and no `thinkingBudget` wherever the host's pi names the model (the `[HOST_LIMITATION]` row above elsewhere). On a host whose `internal/config` waterfall hands listeners the fiber (alpha's npm cell today, cordis `4.0.6-alpha.2`), the claim does not engage yet: a `[KNOWN_DEFECT]` row asserts that pi-ai serves both planes and the claim rows skip by name; the `0.2.2-alpha.3` bump adopts that contract and flips the row. + +## 13. Measured: which endpoint families the gateway serves + +P-R2 probes three URL shapes on both planes, for all four protocols: + +| Shape | Zen | Go | Sent to by | +| :-- | :-- | :-- | :-- | +| legacy | `/zen/v1/…` | `/zen/go/v1/…` | `endpoint: legacy`, except Go's Google | +| root | `/inference/v1/…` | `/inference/go/v1/…` | nobody; measured only. A route-level `baseURL` on it selects modern | +| modern | `/inference/{openai,anthropic,google}/…` | `/inference/go/{openai,anthropic,google}/…` | `auto`, `modern`, and Go's Google under every setting | + +The root shape is the provider-level base `models.opencode.ai` declared for each plane until the catalog regeneration of 2026-10-16 ([engineering notes, "The endpoint chain"](./engineering-notes.md#the-endpoint-chain-read-out-of-the-client-instead-of-inferred)); the catalog now declares the modern completions base (`OPENCODE_PLANE_API` in `src/catalog-data.ts`). Nothing the plugin sends goes there, so its cells are recorded and never asserted on their own: a root path that moves cannot fail a pull request. + +**The default is `modern`, and P-R2 keeps it.** `auto` resolves to `AUTO_ENDPOINT_FAMILY`, and `test/endpoint-family.test.ts` pins it. The flip rule: a keyed modern cell (`*` below) that answers `path-gone`, `wrong-protocol` or `auth-refused` with the plane's own key turns the live job red. Each modern cell is read beside the legacy and root cells of the same plane and protocol, which carry the same model and key, and the run is flip evidence — the evidence for P-R2b, the `fix:` that flips the default, one value, with the copy, `TODAY_DISPATCH` and the re-tagged wire facets in the same change — when the modern cell answers `wrong-protocol`, `path-gone`, `auth-refused` or `model-gone` while the cell a flip would send the same plane and protocol to (the `†` cell, when it is not the modern cell itself) accepted the request (`served` or `path-exists`; beside a refused key or a gone path `account` counts too, since it shows the key was read and the path exists, but not beside a refused protocol or model, since the account's gate answers before the format check and the model's routing). A modern `model-gone` beside any shape that accepted the model (`served` or `path-exists`) turns the job red too: the gateway answers "Model is unavailable" for an id sent where it is not served, so a model another shape accepts is the modern family refusing it. A refusal that only the root, or the `/zen/go` cell of Go's Google (which stays modern under every family), accepted turns the job red but is not flip evidence: no flip reaches that shape. Nor is a refusal no other shape accepted, and each other shape counts only for what it measured: a key refused on every shape that read it is the CI secret, a key no other shape read measured nothing else (a gone path answers before the key is read; a rate limit or a busy upstream may too), a protocol or a path the `†` cell refused too is one a flip would not serve (and Go's Google, which no flip moves, is always that), and a protocol or a path the `†` cell measured nothing about (a rate limit, a busy upstream, the account's gate, no answer, an unreadable one) asks for a re-run — all red, none flip evidence. A keyed modern cell that answers nothing readable — `other`, `redirected`, `session-refused`, `transport-error`, `no-model` — also turns the job red, but says nothing about the family; and a plane none of whose modern cells was `served` (the account's answer, a rate limit, a busy upstream, a retired model no other shape accepted, a parsed but refused body) skips, since it measured nothing. + +**`endpoint: legacy` is recorded, not guarded.** Its cells (`†` below) are probed keyless and keyed and their verdicts printed; the keyless matrix requires a legacy answer to be readable (an unreadable one, or none, is a matrix that measured nothing), but no live assertion reads a `†` cell's path-gone — except Go's Google, which is also a default cell (`* †`) and is guarded as one; what guards the configuration is the offline pin that it maps every protocol on both planes (`test/endpoint-family.test.ts`). + +How to read a cell. Each is the verdict, then the HTTP status: + +- **Keyless** (no credential; one PAID model per plane and protocol): `path-gone` is a 404, the path does not exist; `path-exists` is a 401 or 403, or a 400 that is not about the model; `model-gone` is a 410, a 400 "Model is unavailable", or a refusal of type `ModelDeprecated` at any status (read before the 404, so a withdrawn model never reads as a gone path); `redirected` is a 3xx, which no probe follows, and its cell names the path it pointed at (`redirected (308 → /…)`); `other` is anything else; `no-model` means the live catalog had no paid model for the protocol, so nothing was sent. +- **Every table** (a probe is retried once after a 429, a 5xx or no answer): `rate-limited` is a 429 on the last attempt, or a 429 and then no answer — the gateway's measured rate-limit hang; `transport-error` is no answer on the last attempt, unless an earlier one was a 429 (no answer after a 5xx or a busy upstream is `transport-error` too: only the 429 hang is measured). +- **Free** (one free completions model, keyless, on the default completions path): `served` or `free-tier-gated` (a `403 FreeTierError`: the free tier parsed the request) pass; `rate-limited` skips, and so does `free-usage-limited`, a `429 FreeUsageLimitError` — the tier's usage cap refusing the caller, which is the gate's decision rather than a rate limit, so it is neither retried nor read as served (L3 in the plan's lessons); `wrong-format`, `auth-required`, `path-gone`, `redirected`, `other` (a 500 or 502 included) and `transport-error` fail; `model-gone` and `upstream-busy` (a 503, a 529 or a 5xx whose body names an overload: the free model is the one keyless request that reaches the upstream) move on to the next free model — `model-gone` being a 410, a 400 "Model is unavailable", or a refusal of type `ModelDeprecated` at any status (measured as a 400 for a model the catalog still listed); when every candidate moved on, a busy one among them skips, and the line names that busy candidate's model and status; three retired ones fail; `not run` means the run never reached the probe. +- **Keyed** (the plane's own key, in the header the protocol reads, with the identity headers the fetch patch puts on a turn, its user agent included; on Go that is `OPENCODE_GO_API_KEY`, the key a Go entry naming `apiKeyEnv: OPENCODE_GO_API_KEY` sends — README step 3 — not the Zen key a Go entry without it sends, which these probes do not measure): `served` is a 2xx; `path-exists` a 400 about the content; `wrong-protocol` a `400 ModelProtocolUnsupported`; `auth-refused` a 401 or 403 with the key; `session-refused` the request's session refused (`MissingSessionID`, or a `FreeTierError` on a paid model); `account` no funds or a model the account may not use, which shows the key was read and the path exists, but nothing about whether the path serves the protocol or the model; `path-gone` a 404 whose body names no model, and `model-gone` a refusal of type `ModelDeprecated` at any status, a 410, a 400 "Model is unavailable" or a 404 whose body names a model — Anthropic's `not_found_error` with a `model: …` message, `model_not_found`, or Gemini's `models/ is not found`; Anthropic's bare `not_found_error`, which every Anthropic 404 carries, is `path-gone` (a keyed request can reach the upstream); `upstream-busy` a 503, a 529 or a 5xx whose body names an overload — the upstream was busy, or the gateway's front answered before reading the request, so like a rate limit it neither fails nor passes a plane, and never counts as a shape accepting the request; `redirected` and `other` as keyless, a 500 or 502 included; `no-key` means the run had no key for that plane. +- `*` marks a cell the default family dispatches to (`TODAY_DISPATCH`), `†` one `endpoint: legacy` dispatches to (`LEGACY_DISPATCH`), and `not sent` a cell the run did not reach. + +Three records sit beside the tables, each measured and never asserted (the plan's lessons L4, L5, L9, L10 and L13; P6 and P-R3 act on them): + +- **System One** (keyless, the Zen plane only, no model named): TypeSafe's classification protocol on its three shapes, `/zen/v1/systemone`, `/inference/v1/systemone` and `/inference/systemone/v1/systemone`, read as the keyless matrix reads a cell. The plugin serves no classifier, so nothing dispatches there. +- **Keyed variants** (each plane's default completions cell with its own key): the probe identity (no `x-opencode-request`, `x-opencode-project: global`) beside one change each — a minted `x-opencode-request` in the client's `msg_` shape, `x-opencode-project` as a folder name and as a 40-hex digest — and a paid completions model that reasons, sent with no `reasoning_effort` and with `"none"`. Each answer is read as a keyed cell. +- **`Retry-After`**: the header of every keyed 429, matrix and variants alike, or `absent`. + +The CI `e2e` job writes these tables to its run summary and to the `endpoint-families` artifact (`test/support/endpoint-record.ts` formats them). A run's tables are pasted here with their date and run link; they are never hand-written, and `test/endpoint-record.test.ts` holds this section to the formatter's columns and row order. + +### Keyless · pending the first CI run + +| Plane | Protocol | Model | `/zen(/go)/v1` (legacy) | `/inference(/go)/v1` (root) | `/inference(/go)/{openai,anthropic,google}/…` (modern) | +| :-- | :-- | :-- | :-- | :-- | :-- | +| `opencode` | `openai-completions` | pending | pending † | pending | pending * | +| `opencode` | `openai-responses` | pending | pending † | pending | pending * | +| `opencode` | `anthropic-messages` | pending | pending † | pending | pending * | +| `opencode` | `google-generative-ai` | pending | pending † | pending | pending * | +| `opencode-go` | `openai-completions` | pending | pending † | pending | pending * | +| `opencode-go` | `openai-responses` | pending | pending † | pending | pending * | +| `opencode-go` | `anthropic-messages` | pending | pending † | pending | pending * | +| `opencode-go` | `google-generative-ai` | pending | pending | pending | pending * † | + +Free model on `/inference/openai/v1/chat/completions`: pending the first CI run. + +System One on the Zen plane, keyless (nothing dispatches there): pending the first CI run. + +### Keyed · pending the first CI run + +| Plane | Protocol | Model | `/zen(/go)/v1` (legacy) | `/inference(/go)/v1` (root) | `/inference(/go)/{openai,anthropic,google}/…` (modern) | +| :-- | :-- | :-- | :-- | :-- | :-- | +| `opencode` | `openai-completions` | pending | pending † | pending | pending * | +| `opencode` | `openai-responses` | pending | pending † | pending | pending * | +| `opencode` | `anthropic-messages` | pending | pending † | pending | pending * | +| `opencode` | `google-generative-ai` | pending | pending † | pending | pending * | +| `opencode-go` | `openai-completions` | pending | pending † | pending | pending * | +| `opencode-go` | `openai-responses` | pending | pending † | pending | pending * | +| `opencode-go` | `anthropic-messages` | pending | pending † | pending | pending * | +| `opencode-go` | `google-generative-ai` | pending | pending | pending | pending * † | + +### Keyed variants · pending the first CI run + +| Plane | Probe | Lesson | Model | `/inference(/go)/openai/v1/chat/completions` | +| :-- | :-- | :-- | :-- | :-- | +| `opencode` | the probe identity: no `x-opencode-request`, `x-opencode-project: global` | L4, L5 | pending | pending | +| `opencode` | `x-opencode-request: msg_…` | L4 | pending | pending | +| `opencode` | `x-opencode-project: ` | L5 | pending | pending | +| `opencode` | `x-opencode-project: <40 hex>` | L5 | pending | pending | +| `opencode` | a reasoning model, no `reasoning_effort` | L10 | pending | pending | +| `opencode` | `reasoning_effort: "none"` | L10 | pending | pending | +| `opencode-go` | the probe identity: no `x-opencode-request`, `x-opencode-project: global` | L4, L5 | pending | pending | +| `opencode-go` | `x-opencode-request: msg_…` | L4 | pending | pending | +| `opencode-go` | `x-opencode-project: ` | L5 | pending | pending | +| `opencode-go` | `x-opencode-project: <40 hex>` | L5 | pending | pending | +| `opencode-go` | a reasoning model, no `reasoning_effort` | L10 | pending | pending | +| `opencode-go` | `reasoning_effort: "none"` | L10 | pending | pending | + +`Retry-After` on a keyed 429: pending the first CI run. diff --git a/docs/testing.md b/docs/testing.md index 6109fad..f8bc548 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -37,7 +37,7 @@ Three properties the unit and mock suites keep by construction rather than by ca **Three rules, learned from failures:** -1. **A live case skips on a vendor-side move, it does not fail.** A retired model (`410 ModelDeprecated`, or an upstream's `400 … Model is unavailable`), a closed tier (`FreeTierError`), a missing entitlement and a busy gateway (`429`) are all "not observable today". The reason is printed so the run says which. +1. **A live case skips on a vendor-side move, it does not fail.** A retired model (`410 ModelDeprecated`, or an upstream's `400 … Model is unavailable`), a closed tier (`FreeTierError`, or the free tier's usage cap, `429 FreeUsageLimitError`), a missing entitlement and a busy gateway (any other `429`) are all "not observable today". The reason is printed so the run says which. 2. **A case that walks candidates walks past the retired ones.** A `410` is an answer, so a picker that accepts "anything but a refusal" picks a dead model and then fails on it. For a `400` it is the body, not the status, that makes it a retirement: a bare `400` is also a malformed request, which is the plugin's fault and must still fail (`test/e2e/verdict.ts`, `retiredAtTheGateway`). 3. **An assertion about the vendor is a claim about a moving thing.** Prefer asserting the PLUGIN's behaviour (which endpoint it chose, which headers it sent) over the vendor's answer. diff --git a/src/utils/plane-model.ts b/src/utils/plane-model.ts index 2323aed..55fe7d6 100644 --- a/src/utils/plane-model.ts +++ b/src/utils/plane-model.ts @@ -8,7 +8,7 @@ * | name, context, max tokens, modalities | the user's row > pi-ai's `modelOverrides` > catalog > pi > pi-ai's own route defaults, as its `Config` filled them (D17) | * | cost | catalog > pi > zero | * | reasoning | the user's `reasoningEfforts` > the override's > the catalog ({@link reasoningFor}, D1) > pi > none | - * | baseUrl | the route's `baseURL`, shaped > the endpoint family table ({@link baseUrlFor}) | + * | baseUrl | the route's `baseURL`, shaped > the endpoint family table ({@link baseUrlFor}; `auto` is {@link AUTO_ENDPOINT_FAMILY}) | * * A route-level `api` never reclassifies an id the catalog describes: the * plane serves every protocol at once, so the route's `api` only says what an @@ -45,9 +45,19 @@ import { protocolForSdk, } from "./protocols.ts"; -/** The endpoint family a plane sends to; `auto` is `modern`. */ +/** The endpoint family a plane sends to; `auto` is {@link AUTO_ENDPOINT_FAMILY}. */ export type EndpointChoice = "auto" | "legacy" | "modern"; +/** + * The family `auto` — the default `endpoint` (`DEFAULT_ENDPOINT_FAMILY`) — + * sends to. It has been `modern` since P-R1; P-R2 measures the families on the + * live gateway (`docs/redirect-planes.md` §13) and keeps it. The family `auto` + * resolves to is this one value; a flip also moves the copy, `TODAY_DISPATCH`, + * the bases `test/endpoint-family.test.ts` pins and the re-tagged wire facets + * (§13). + */ +export const AUTO_ENDPOINT_FAMILY: Exclude = "modern"; + /** The catalog spec fields this module reads (`CatalogModelSpec`'s shape). */ export interface PlaneCatalogSpec extends ReasoningSpec { readonly id: string; @@ -184,7 +194,7 @@ const forProtocol = (protocol: Protocol, base: string): string => * the legacy one, `/inference(/go)(/openai)/v1` the modern one. * 2. Any other route-level `baseURL` is the user's endpoint, used as written * (trailing slashes trimmed, and a trailing `/v1` dropped for Anthropic). - * 3. Otherwise the `endpoint` choice; `auto` is `modern`. + * 3. Otherwise the `endpoint` choice; `auto` is {@link AUTO_ENDPOINT_FAMILY}. * 4. Modern: the catalog's family map for the model's SDK, else for the * protocol's family, else the measured fallback. * 5. Legacy: the fixed table — Go's Google cell included. @@ -202,7 +212,8 @@ export const baseUrlFor = ( familyApi: Readonly>, routeBaseURL?: string ): string => { - let family: "legacy" | "modern" = endpoint === "legacy" ? "legacy" : "modern"; + let family: Exclude = + endpoint === "auto" ? AUTO_ENDPOINT_FAMILY : endpoint; if (routeBaseURL !== undefined && routeBaseURL.length > 0) { const path = vendorPathOf(routeBaseURL); if (path?.startsWith("/zen") === true) { diff --git a/test/e2e/dispatch-levels.e2e.ts b/test/e2e/dispatch-levels.e2e.ts index d679c42..4e26d9c 100644 --- a/test/e2e/dispatch-levels.e2e.ts +++ b/test/e2e/dispatch-levels.e2e.ts @@ -25,9 +25,11 @@ * gateway that refuses it fails here rather than in a user's turn. Each * protocol walks up to three toggle-only candidates in catalog order, moving * past one refused on entitlement or one the vendor withdrew (confirmed by a - * raw probe of the same id; unconfirmed, the turn fails). It skips without - * the plane's key, and, naming the protocol, when a protocol's every - * candidate was refused on entitlement or withdrawn. + * raw probe of the same id; unconfirmed, the turn fails). It runs on the + * `full` tier of `OPENCODE_E2E_KEYED` (the default when it is unset; CI runs + * it nightly and on dispatch), and skips without the plane's key, on a lower + * tier, and, naming the protocol, when a protocol's every candidate was + * refused on entitlement or withdrawn. * * Opt-in (`OPENCODE_E2E=1`), like every file in this suite. The sweep needs * no network and no port: its value is the real rule and the real catalog. @@ -44,6 +46,7 @@ import { mountTierA, waitForPlaneOwner, } from "../support/host.ts"; +import { armedKey, keyedTierOf } from "../support/keyed-probes.ts"; import { isModelGoneMessage } from "../support/keyless-matrix.ts"; import { accessDisabled, @@ -68,19 +71,21 @@ import { } from "../support/src.ts"; const ENABLED = process.env.OPENCODE_E2E === "1"; +/** What may spend a key: the live block runs on the `full` tier only. */ +const TIER = keyedTierOf(process.env.OPENCODE_E2E_KEYED); /** The two plane routes, the catalog and family map each resolves from, and its key. */ const PLANES = [ { catalog: OPENCODE_ZEN_CATALOG, family: OPENCODE_FAMILY_API.zen, - key: process.env.OPENCODE_API_KEY, + key: armedKey(TIER, "full", process.env.OPENCODE_API_KEY), route: "opencode", }, { catalog: OPENCODE_GO_CATALOG, family: OPENCODE_FAMILY_API.go, - key: process.env.OPENCODE_GO_API_KEY, + key: armedKey(TIER, "full", process.env.OPENCODE_GO_API_KEY), route: "opencode-go", }, ] as const; diff --git a/test/e2e/keyed-probes.e2e.ts b/test/e2e/keyed-probes.e2e.ts new file mode 100644 index 0000000..25b80e1 --- /dev/null +++ b/test/e2e/keyed-probes.e2e.ts @@ -0,0 +1,412 @@ +/** + * The minimal keyed probes against the live gateway: one request per plane × + * protocol × URL shape, each with the plane's OWN key in the header its + * protocol reads, and a body of a few output tokens. + * + * The keyless matrix (`keyless-matrix.e2e.ts`) can only say whether a path + * exists: the gateway authenticates before it routes, so a paid model answers + * 401 everywhere without a key. With the key the answer says whether the path + * SERVES the protocol for that plane's account, which is what the default + * family rests on. + * + * Each keyed cell carries `PROBE_IDENTITY` (`test/support/keyed-probes.ts`), + * the identity headers the fetch patch puts on a turn (the turn's user agent + * `OPENCODE_UA`, `x-opencode-client`, `x-opencode-project`, an in-band session + * id under both session headers), so the request is the plugin's request: the + * Go plane refuses a keyed request with no session header, and the gateway + * reads the user agent as origin proof. `keyedCell` applies it, and the + * offline suite holds that same constant to what the real fetch patch sends, + * so this file has no identity of its own to drift. The one turn header it + * leaves out is `x-opencode-request`, a per-request metric id. + * + * What it decides, per plane, from the cells the default family dispatches to + * (`TODAY_DISPATCH`), each read beside the legacy and root cells of the same + * protocol, which carry the same model and key (`planeOutcome`): + * + * - FAIL when one answers path-gone, wrong-protocol or auth-refused, or a + * retired model (`model-gone`) that another shape accepted, or an answer + * that measured nothing a turn could use: an unreadable status (`other`), a + * redirect, a refused session, no answer on the last attempt that did not + * follow a 429 (`transport-error`), or no model to send (`no-model`). The + * flip rule's evidence (`docs/redirect-planes.md` §13) is a wrong-protocol, + * a path-gone, an auth-refused or a model-gone that the cell a flip would + * send the same protocol to (`LEGACY_DISPATCH`'s, when it is not the + * default cell itself, so never Go's Google) accepted. A refusal only the + * root or Go Google's `/zen/go` cell accepted fails but is not evidence, as + * no flip reaches them; nor is a key refused on every shape that read it + * (the CI secret), a key no other shape read, or a protocol or a path the + * flip's cell refused too or measured nothing about (a re-run). Each other + * shape counts only for what it measured: the account's answer shows a key + * read and a path that exists, never a protocol or a model served. + * - SKIP when none of them was `served`: the account's answer (no funds, a + * model the account may not use), a rate limit (a 429, or a 429 and then no + * answer, the measured rate-limit hang), a busy upstream (a 503, a 529, or a + * 5xx naming an overload — never read as a shape accepting the request, + * since the gateway's front may answer it), a retired model, or a + * parsed but refused body. Nothing showed the plugin's turns are served, so + * it is a skip, never a pass. + * - PASS otherwise. + * + * Every other cell — the legacy cells `endpoint: legacy` dispatches to, which + * P-R2 decided are recorded, not guarded, and the root shape nothing + * dispatches to — never fails or passes a plane on its own: it is recorded, + * and read only beside a default cell of the same protocol. + * + * A second case per plane sends the keyed VARIANTS of the default + * completions cell (`keyedVariantCells`, lessons L4, L5 and L10): the probe + * identity beside a minted `x-opencode-request`, `x-opencode-project` as a + * folder name and as a 40-hex digest, and a paid completions model that + * reasons, with no `reasoning_effort` and with `"none"`. Every 429 either case + * meets has its `Retry-After` noted (lesson L9). All of it is recorded — + * the variants table and the `Retry-After` line follow the keyed table in + * `keyed.md` — and none of it is asserted: P6 and P-R3 act on what it shows. + * + * Armed by the `probes` tier (`OPENCODE_E2E_KEYED`, unset meaning `full`): + * CI runs it on pull requests that touch `src/host/**`, the modules that + * decide the legacy URLs and protocols the probes send or the family the + * plugin dispatches to, the CLI version in the user agent and the + * `anthropic-version` the probes carry, or the probes' own inputs (the exact + * list, and what it leaves out, is the tier step in `.github/workflows/ci.yml`); + * nightly and on dispatch. + * It skips without either key, and a plane without its key is reported + * skipped. Every verdict goes into a dated table + * (`test/support/endpoint-record.ts`), printed and, when + * `OPENCODE_E2E_RECORD_DIR` is set, written there as `keyed.md` — rewritten + * after every cell, so a job killed at its time limit still leaves what it + * measured, the cells it did not reach reading `not sent`. A redirect is never + * followed (`redirect: "manual"`, which `test/keyed-probes.test.ts` holds + * every probe `fetch` to): it is recorded as `redirected`, with the path it + * pointed at, in the console table and the dated record alike. No + * header and no key is ever printed; a body is printed only through + * `excerpt`, which redacts the key. + * + * Like the keyless matrix it measures only the vendor's own paths, so it skips + * while a mirror base URL is set. The cells, the tier and the classifier are + * unit-tested offline in `test/keyed-probes.test.ts`. + * + * OPENCODE_E2E=1 OPENCODE_API_KEY=… OPENCODE_GO_API_KEY=… pnpm run test:e2e + * + * @module test/e2e/keyed-probes.e2e + */ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { + type RecordedProbe, + type RecordedVariant, + recordTable, + retryAfterLine, + runReference, + variantTable, +} from "../support/endpoint-record.ts"; +import { + armedKey, + excerpt, + isArmed, + KEYED_VARIANTS, + keyedCell, + keyedTierOf, + keyedVariantCells, + keyedVerdictOf, + planeOutcome, + reasoningCompletionsModel, + retryAfterOf, +} from "../support/keyed-probes.ts"; +import { + cellKey, + isTransientProbeAnswer, + keylessMatrix, + type MatrixCell, + type ProbeAnswer, + redirectPathOf, +} from "../support/keyless-matrix.ts"; +import { + type CatalogModelSpec, + MODELS_OPENCODE_URL, + parseModelsDevCatalog, +} from "../support/src.ts"; + +const LIVE = process.env.OPENCODE_E2E === "1"; +/** A mirror the rest of the suite is pointed at, which this file cannot follow. */ +const MIRRORED = ["OPENCODE_ZEN_BASE_URL", "OPENCODE_GO_BASE_URL"].some( + (name) => (process.env[name] ?? "").length > 0 +); +const TIER = keyedTierOf(process.env.OPENCODE_E2E_KEYED); +const KEYS = { + opencode: armedKey(TIER, "probes", process.env.OPENCODE_API_KEY), + "opencode-go": armedKey(TIER, "probes", process.env.OPENCODE_GO_API_KEY), +} as const; + +/** Per request: protocol-routing's measured budget, so a hung gateway cannot outlast the job. */ +const PROBE_TIMEOUT_MS = 20_000; +/** The pause before the one retry of a transient answer. */ +const RETRY_AFTER_MS = 2000; + +/** + * One keyed request: its status, body and redirect path, or `undefined` on a + * transport error. A redirect is not followed: a moved path must not be + * recorded as its target's answer, and a cross-origin hop would carry the + * `x-api-key` / `x-goog-api-key` header with it. + */ +const send = async (cell: MatrixCell): Promise => { + try { + const response = await fetch(cell.url, { + body: JSON.stringify(cell.body), + headers: cell.headers, + method: cell.method, + redirect: "manual", + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + const location = redirectPathOf(response.headers.get("location"), cell.url); + const retryAfter = response.headers.get("retry-after"); + return { + bodyText: await response.text(), + status: response.status, + ...(location === undefined ? {} : { location }), + ...(retryAfter === null ? {} : { retryAfter }), + }; + } catch { + return undefined; + } +}; + +/** {@link send}, retried once on a rate limit, a server error or a dropped connection: every attempt's answer. */ +const sendWithRetry = async ( + cell: MatrixCell +): Promise<(ProbeAnswer | undefined)[]> => { + const answer = await send(cell); + if (!isTransientProbeAnswer(answer?.status, answer?.bodyText)) { + return [answer]; + } + await new Promise((resolve) => { + setTimeout(resolve, RETRY_AFTER_MS); + }); + return [answer, await send(cell)]; +}; + +/** One probe's outcome, as the console table prints it: no header, no key. */ +interface KeyedRow { + readonly cell: string; + readonly model: string | undefined; + readonly status: number | undefined; + readonly verdict: string; + /** The redacted start of the body, kept for a verdict other than `served`. */ + readonly body?: string; + /** The path a redirect pointed at. */ + readonly location?: string; +} + +describe.skipIf( + !LIVE || + MIRRORED || + !isArmed(TIER, "probes") || + (KEYS.opencode === undefined && KEYS["opencode-go"] === undefined) +)("live gateway · the keyed probes", () => { + let cells: MatrixCell[] = []; + let slices: Readonly< + Record<"opencode" | "opencode-go", readonly CatalogModelSpec[]> + > = { + opencode: [], + "opencode-go": [], + }; + const recorded: RecordedProbe[] = []; + const variants: RecordedVariant[] = []; + /** Every keyed 429 and its `Retry-After` (lesson L9), matrix and variants alike. */ + const retryAfters: { cell: string; value: string | null }[] = []; + + /** {@link sendWithRetry}, noting the `Retry-After` of every 429 it met. */ + const sendNoting = async ( + cell: MatrixCell, + name: string + ): Promise<(ProbeAnswer | undefined)[]> => { + const attempts = await sendWithRetry(cell); + retryAfters.push( + ...retryAfterOf(attempts).map((value) => ({ cell: name, value })) + ); + return attempts; + }; + + beforeAll(async () => { + const listing = await fetch(MODELS_OPENCODE_URL, { + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + expect(listing.status).toBe(200); + const parsed = parseModelsDevCatalog(await listing.json()); + cells = keylessMatrix({ go: parsed.go, zen: parsed.zen }); + slices = { opencode: parsed.zen, "opencode-go": parsed.go }; + }, PROBE_TIMEOUT_MS + 5000); + + /** Print and write the record: after every cell, so a killed job keeps what it measured. */ + const persist = async (print: boolean): Promise => { + const date = new Date().toISOString().slice(0, 10); + const run = runReference({ + repository: process.env.GITHUB_REPOSITORY, + runId: process.env.GITHUB_RUN_ID, + server: process.env.GITHUB_SERVER_URL, + }); + const record = [ + recordTable({ date, probes: recorded, run, title: "Keyed" }), + "", + variantTable({ date, probes: variants, run }), + "", + retryAfterLine(retryAfters), + "", + ].join("\n"); + if (print) { + console.info(record); + } + const dir = process.env.OPENCODE_E2E_RECORD_DIR ?? ""; + if (dir.length > 0) { + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, "keyed.md"), record); + } + }; + + afterAll(async () => { + await persist(true); + }); + + it.for(["opencode", "opencode-go"] as const)( + "%s: no cell the default sends to breaks with the plane's own key, and one is served; the other shapes are recorded", + { + timeout: 12 * 2 * (PROBE_TIMEOUT_MS + RETRY_AFTER_MS) + PROBE_TIMEOUT_MS, + }, + async (plane, context) => { + const planeCells = cells.filter((cell) => cell.plane === plane); + expect(planeCells).toHaveLength(12); + const key = KEYS[plane]; + if (key === undefined) { + recorded.push( + ...planeCells.map((cell) => ({ + family: cell.family, + model: cell.model, + plane, + protocol: cell.protocol, + status: undefined, + verdict: "no-key", + })) + ); + await persist(false); + context.skip(true, "no key for this plane"); + return; + } + const rows: KeyedRow[] = []; + for (const cell of planeCells) { + const name = cellKey(cell); + if (cell.model === undefined) { + rows.push({ + cell: name, + model: undefined, + status: undefined, + verdict: "no-model", + }); + } else { + const attempts = await sendNoting(keyedCell(cell, key), name); + const answer = attempts.at(-1); + const verdict = keyedVerdictOf(attempts); + rows.push({ + cell: name, + model: cell.model, + status: answer?.status, + verdict, + ...(verdict === "served" || answer === undefined + ? {} + : { body: excerpt(answer.bodyText, key) }), + ...(answer?.location === undefined + ? {} + : { location: answer.location }), + }); + } + const row = rows.at(-1); + recorded.push({ + family: cell.family, + model: cell.model, + plane, + protocol: cell.protocol, + status: row?.status, + verdict: row?.verdict ?? "no-model", + ...(row?.location === undefined ? {} : { location: row.location }), + }); + await persist(false); + } + console.table(rows); + const outcome = planeOutcome(rows); + if (outcome.kind === "skip") { + context.skip(true, outcome.reason); + return; + } + expect( + outcome, + outcome.kind === "fail" + ? `${outcome.reason}: ${JSON.stringify(outcome.broken)}` + : undefined + ).toEqual({ kind: "pass" }); + } + ); + + it.for(["opencode", "opencode-go"] as const)( + "%s: records the header and reasoning variants on the default completions cell (L4, L5, L10)", + { + timeout: + KEYED_VARIANTS.length * 2 * (PROBE_TIMEOUT_MS + RETRY_AFTER_MS) + + PROBE_TIMEOUT_MS, + }, + async (plane, context) => { + const key = KEYS[plane]; + if (key === undefined) { + context.skip(true, "no key for this plane"); + return; + } + const representative = cells.find( + (cell) => + cell.plane === plane && + cell.protocol === "openai-completions" && + cell.family === "modern" + )?.model; + const variantCells = keyedVariantCells( + plane, + { reasoning: reasoningCompletionsModel(slices[plane]), representative }, + key + ); + // Recorded, never asserted: the variants measure what P6 and P-R3 + // decide, and no answer here says anything about the default family. + const rows: KeyedRow[] = []; + for (const cell of variantCells) { + const name = `${plane} ${cell.variant.id}`; + let row: KeyedRow = { + cell: name, + model: undefined, + status: undefined, + verdict: "no-model", + }; + if (cell.body !== undefined) { + const attempts = await sendNoting(cell, name); + const answer = attempts.at(-1); + const verdict = keyedVerdictOf(attempts); + row = { + cell: name, + model: cell.model, + status: answer?.status, + verdict, + ...(verdict === "served" || answer === undefined + ? {} + : { body: excerpt(answer.bodyText, key) }), + }; + } + rows.push(row); + variants.push({ + model: row.model, + plane, + status: row.status, + variant: cell.variant.id, + verdict: row.verdict, + }); + await persist(false); + } + console.table(rows); + expect(rows).toHaveLength(KEYED_VARIANTS.length); + } + ); +}); diff --git a/test/e2e/keyless-matrix.e2e.ts b/test/e2e/keyless-matrix.e2e.ts index 56c52fa..ca42a82 100644 --- a/test/e2e/keyless-matrix.e2e.ts +++ b/test/e2e/keyless-matrix.e2e.ts @@ -1,9 +1,11 @@ /** - * The keyless 401-vs-404 matrix against the live gateway. + * The keyless 401-vs-404 matrix against the live gateway, and the keyless + * free-model probe. * - * Sixteen POSTs, none carrying a key: two planes × four protocols × the - * legacy (`/zen(/go)/v1`) and modern (`/inference(/go)/…`) endpoint families, - * each with one PAID model from the live catalog slice (a free model answers + * Twenty-four POSTs, none carrying a key: two planes × four protocols × three + * URL shapes — legacy (`/zen(/go)/v1`), the root (`/inference(/go)/v1`, the + * provider-level base the catalog declared until 2026-10-16) and modern + * (`/inference(/go)/{openai,anthropic,google}/…`) — each with one PAID model from the live catalog slice (a free model answers * keyless with a real completion, so the matrix never picks one). The gateway * authenticates before it routes, so the answer says whether the PATH exists * without spending anything. It runs on every pull request, because it needs @@ -11,21 +13,46 @@ * * What it asserts: * - * - every one of the sixteen cells has a model and was SENT, and every answer - * is one the matrix can read — an empty or reshaped catalog fails here - * rather than measuring nothing; + * - every one of the twenty-four cells has a model and was SENT, and every + * legacy and modern answer is one the matrix can read (`unreadRows`) — an + * empty or reshaped catalog, a redirect, a rate limit the retry did not + * outlast, or no answer fails here rather than measuring nothing. A root + * answer is recorded whatever it is: nothing dispatches there; * - no cell the plugin dispatches to with the default endpoint (`auto`, an * alias of `modern`; `TODAY_DISPATCH`: all four protocols on the modern - * family, on both planes) answers "path gone". + * family, on both planes) answers "path gone" (`goneDefaultRows`); + * - a FREE completions model, sent keyless to the default completions path + * (`/inference/openai/v1/chat/completions`), is parsed there: served, or + * gated by the free tier (`freeProbeOutcome`). A retired free model or a + * busy upstream (a 503, a 529, or a 5xx naming an overload) moves on to the + * next, three at most (`runFreeProbe`, which records the candidate that + * decided, with its own model and status); a rate + * limit (a 429, or a 429 and then no answer) SKIPS the case, as + * `protocol-routing.e2e.ts`'s free cases do, and so does a busy upstream + * that no candidate got past. So does the free tier's usage cap, a + * `429 FreeUsageLimitError`, which is its own verdict + * (`free-usage-limited`): the gate refusing the caller, never retried and + * never read as served (lesson L3). * - * The other legacy verdicts are recorded, not asserted: P-R2 decides the - * family from them. They include the cells an `endpoint: legacy` user - * dispatches to — Responses and Messages on both planes, and Zen's Google — - * so this file does not guard that configuration; P-R2 decides whether it - * should. Go's Google stays on `/inference/go/google/v1beta` under every - * family (`GO_GOOGLE_BASE_URL`), the modern cell `TODAY_DISPATCH` asserts. - * The cell table, the dispatch set and the classifier are unit-tested offline in - * `test/keyless-matrix.test.ts`. + * Beside them it records the System One family on the Zen plane + * (`systemOneCells`: `/zen/v1/systemone`, `/inference/v1/systemone`, + * `/inference/systemone/v1/systemone`, lesson L13), keyless and naming no + * model, and asserts nothing about it: the plugin serves no classifier. + * + * `endpoint: legacy` is recorded, not guarded (P-R2): its cells + * (`LEGACY_DISPATCH`) must answer readably, like every legacy cell, but their + * path-gone is printed and asserted nowhere live — except Go's Google, which + * is also a default cell (`TODAY_DISPATCH`) and is guarded as one; the + * offline pin in `test/endpoint-family.test.ts` guards its mapping. Every + * verdict goes into a dated table (`test/support/endpoint-record.ts`), + * printed and, when `OPENCODE_E2E_RECORD_DIR` is set, written there as + * `keyless.md` for the CI job to publish — rewritten after every cell, so a + * job killed at its time limit still leaves what it measured. A redirect is + * never followed (`redirect: "manual"`, which `test/keyed-probes.test.ts` + * holds every probe `fetch` to): it is recorded as `redirected`, with the + * path it pointed at, in the console table and the dated record alike. The + * cell table, the sets, the classifiers and every decision above are + * unit-tested offline in `test/keyless-matrix.test.ts`. * * It always measures the vendor's own gateway: the cells are the vendor's * paths, which a mirror's `/zen(/go)/v1` base cannot name. So it SKIPS when @@ -36,18 +63,44 @@ * * @module test/e2e/keyless-matrix.e2e */ -import { describe, expect, it } from "vitest"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { + freeProbeLine, + type RecordedProbe, + recordTable, + runReference, + systemOneLine, +} from "../support/endpoint-record.ts"; +import { + type AttemptVerdict, cellKey, classifyKeylessProbe, + freeProbeCells, + freeProbeOutcome, + goneDefaultRows, isTransientProbeAnswer, type KeylessVerdict, keylessMatrix, + MATRIX_FAMILIES, + MATRIX_PROTOCOLS, type MatrixCell, - TODAY_DISPATCH, + type MatrixFamily, + type ProbeAnswer, + redirectPathOf, + runFreeProbe, + systemOneCells, + unreadRows, + verdictOfAttempts, } from "../support/keyless-matrix.ts"; -import { MODELS_OPENCODE_URL, parseModelsDevCatalog } from "../support/src.ts"; +import { + type CatalogModelSpec, + MODELS_OPENCODE_URL, + parseModelsDevCatalog, +} from "../support/src.ts"; const LIVE = process.env.OPENCODE_E2E === "1"; /** A mirror the rest of the suite is pointed at, which this file cannot follow. */ @@ -60,92 +113,176 @@ if (LIVE && MIRRORED) { ); } -const PROBE_TIMEOUT_MS = 30_000; +/** Per request: protocol-routing's measured budget, so a hung gateway cannot outlast the job. */ +const PROBE_TIMEOUT_MS = 20_000; /** The pause before the one retry of a transient answer. */ const RETRY_AFTER_MS = 2000; +/** How many free models the free probe tries before it gives up. */ +const FREE_CANDIDATES = 3; /** One probe's outcome, with no request or response header in it. */ interface ProbeRow { readonly cell: string; + readonly family: MatrixFamily; readonly model: string | undefined; readonly status: number | undefined; - readonly verdict: KeylessVerdict | "transport-error"; + readonly verdict: AttemptVerdict | KeylessVerdict; /** The start of the body, kept only for a verdict the matrix cannot read. */ readonly body?: string; + /** The path a redirect pointed at. */ + readonly location?: string; } -/** One keyless request: its status and body, or `undefined` on a transport error. */ +/** + * One keyless request: its status, body and redirect path, or `undefined` on + * a transport error. A redirect is not followed, so a moved path is not + * recorded as its target's answer. + */ const send = async ( - cell: MatrixCell -): Promise<{ status: number; bodyText: string } | undefined> => { + cell: Pick +): Promise => { try { const response = await fetch(cell.url, { body: JSON.stringify(cell.body), headers: cell.headers, method: cell.method, + redirect: "manual", signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), }); - return { bodyText: await response.text(), status: response.status }; + const location = redirectPathOf(response.headers.get("location"), cell.url); + return { + bodyText: await response.text(), + status: response.status, + ...(location === undefined ? {} : { location }), + }; } catch { return undefined; } }; +/** {@link send}, retried once on a rate limit, a server error or a dropped connection: every attempt's answer. */ +const sendWithRetry = async ( + cell: Pick +): Promise<(ProbeAnswer | undefined)[]> => { + const answer = await send(cell); + // One transient answer must not fail every pull request. + if (!isTransientProbeAnswer(answer?.status, answer?.bodyText)) { + return [answer]; + } + await new Promise((resolve) => { + setTimeout(resolve, RETRY_AFTER_MS); + }); + return [answer, await send(cell)]; +}; + const probe = async (cell: MatrixCell): Promise => { const name = cellKey(cell); if (cell.body === undefined) { return { cell: name, + family: cell.family, model: undefined, status: undefined, verdict: "no-model", }; } - let answer = await send(cell); - // A rate limit, a server error or a dropped connection is retried once - // before it is read: one transient answer must not fail every pull request. - if (isTransientProbeAnswer(answer?.status)) { - await new Promise((resolve) => { - setTimeout(resolve, RETRY_AFTER_MS); - }); - answer = await send(cell); - } - if (answer === undefined) { - return { - cell: name, - model: cell.model, - status: undefined, - verdict: "transport-error", - }; - } - const verdict = classifyKeylessProbe(answer); + const attempts = await sendWithRetry(cell); + const answer = attempts.at(-1); + const verdict = verdictOfAttempts(attempts, classifyKeylessProbe); return { cell: name, + family: cell.family, model: cell.model, - status: answer.status, + status: answer?.status, verdict, - ...(verdict === "other" ? { body: answer.bodyText.slice(0, 200) } : {}), + ...(verdict === "other" && answer !== undefined + ? { body: answer.bodyText.slice(0, 200) } + : {}), + ...(answer?.location === undefined ? {} : { location: answer.location }), }; }; describe.skipIf(!LIVE || MIRRORED)( "live gateway · the keyless path matrix", () => { + let zen: readonly CatalogModelSpec[] = []; + let go: readonly CatalogModelSpec[] = []; + const recorded: RecordedProbe[] = []; + let free: { + model: string | undefined; + status: number | undefined; + verdict: string; + } = { model: undefined, status: undefined, verdict: "not run" }; + const systemOne: { + family: MatrixFamily; + status: number | undefined; + verdict: string; + }[] = []; + + beforeAll(async () => { + const listing = await fetch(MODELS_OPENCODE_URL, { + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + expect(listing.status).toBe(200); + // The parsed slices have already dropped their deprecated rows, so no + // `isDeprecated` predicate is passed: it could never exclude an id here. + ({ go, zen } = parseModelsDevCatalog(await listing.json())); + }, PROBE_TIMEOUT_MS + 5000); + + /** Print and write the record: after every cell, so a killed job keeps what it measured. */ + const persist = async (print: boolean): Promise => { + const record = [ + recordTable({ + date: new Date().toISOString().slice(0, 10), + probes: recorded, + run: runReference({ + repository: process.env.GITHUB_REPOSITORY, + runId: process.env.GITHUB_RUN_ID, + server: process.env.GITHUB_SERVER_URL, + }), + title: "Keyless", + }), + "", + freeProbeLine(free), + "", + systemOneLine(systemOne), + "", + ].join("\n"); + if (print) { + console.info(record); + } + const dir = process.env.OPENCODE_E2E_RECORD_DIR ?? ""; + if (dir.length > 0) { + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, "keyless.md"), record); + } + }; + + afterAll(async () => { + await persist(true); + }); + it( - "sends every cell, reads every answer, and no cell the plugin dispatches to (or P-R1 will) answers path-gone", + "sends every cell, reads every answer the plugin can dispatch to, and no cell the default dispatches to answers path-gone", async () => { - const listing = await fetch(MODELS_OPENCODE_URL, { - signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), - }); - expect(listing.status).toBe(200); - const parsed = parseModelsDevCatalog(await listing.json()); - // The parsed slices have already dropped their deprecated rows, so no - // `isDeprecated` predicate is passed: it could never exclude an id here. - const cells = keylessMatrix({ go: parsed.go, zen: parsed.zen }); - expect(cells).toHaveLength(16); + const cells = keylessMatrix({ go, zen }); + expect(cells).toHaveLength( + 2 * MATRIX_PROTOCOLS.length * MATRIX_FAMILIES.length + ); const rows: ProbeRow[] = []; for (const cell of cells) { - rows.push(await probe(cell)); + const row = await probe(cell); + rows.push(row); + recorded.push({ + family: cell.family, + model: row.model, + plane: cell.plane, + protocol: cell.protocol, + status: row.status, + verdict: row.verdict, + ...(row.location === undefined ? {} : { location: row.location }), + }); + await persist(false); } console.table(rows); // A model-less cell sends nothing. On the bundled catalogs every cell @@ -154,21 +291,68 @@ describe.skipIf(!LIVE || MIRRORED)( const unsent = rows.filter((row) => row.verdict === "no-model"); expect(unsent, JSON.stringify(unsent)).toEqual([]); // A 200 stays unreadable on purpose: it means a free model slipped past - // the matrix's filter and a completion was spent. - const unread = rows.filter( - (row) => - !["path-exists", "model-gone", "path-gone"].includes(row.verdict) - ); + // the matrix's filter and a completion was spent. The root shape is + // recorded, not read: nothing dispatches there, so whatever it answers + // cannot fail a pull request. + const unread = unreadRows(rows); expect(unread, JSON.stringify(unread)).toEqual([]); - const goneToday = rows.filter( - (row) => TODAY_DISPATCH.has(row.cell) && row.verdict === "path-gone" - ); + const goneToday = goneDefaultRows(rows); expect( goneToday, `a path the plugin dispatches to with the default endpoint, on every host, is gone: ${JSON.stringify(goneToday)}` ).toEqual([]); }, - 16 * 2 * (PROBE_TIMEOUT_MS + RETRY_AFTER_MS) + PROBE_TIMEOUT_MS + 24 * 2 * (PROBE_TIMEOUT_MS + RETRY_AFTER_MS) + PROBE_TIMEOUT_MS + ); + + it( + "a free model on the default completions path is parsed there, without a key", + async (context) => { + const cells = freeProbeCells(zen).slice(0, FREE_CANDIDATES); + expect( + cells.length, + "the live Zen slice has no free completions model" + ).toBeGreaterThan(0); + // The walk moves past a retired model or a busy upstream, and the + // candidate that decides is recorded with its own model and status. + const { decided } = await runFreeProbe(cells, sendWithRetry); + const { bodyText, model, status, verdict } = decided; + free = { model, status, verdict }; + await persist(false); + const outcome = freeProbeOutcome(verdict); + const detail = `free model ${model ?? ""} on the default completions path: ${verdict} (${status ?? "no answer"}) ${bodyText.slice(0, 200)}`; + if (outcome === "skip") { + context.skip( + true, + verdict === "free-usage-limited" + ? `the free tier's usage cap answered, which measures the caller, not the path: ${detail}` + : `the gateway was rate-limiting or the upstream busy: ${detail}` + ); + return; + } + expect(outcome, detail).toBe("pass"); + }, + FREE_CANDIDATES * 2 * (PROBE_TIMEOUT_MS + RETRY_AFTER_MS) + + PROBE_TIMEOUT_MS + ); + + it( + "records which shapes the System One family answers on (L13)", + async () => { + // Recorded, never asserted: the plugin serves no classifier, so no + // answer here can break a turn. + for (const cell of systemOneCells()) { + const attempts = await sendWithRetry(cell); + systemOne.push({ + family: cell.family, + status: attempts.at(-1)?.status, + verdict: verdictOfAttempts(attempts, classifyKeylessProbe), + }); + await persist(false); + } + expect(systemOne).toHaveLength(MATRIX_FAMILIES.length); + }, + 3 * 2 * (PROBE_TIMEOUT_MS + RETRY_AFTER_MS) + PROBE_TIMEOUT_MS ); } ); diff --git a/test/e2e/opencode-live.e2e.ts b/test/e2e/opencode-live.e2e.ts index cdd55d2..156797d 100644 --- a/test/e2e/opencode-live.e2e.ts +++ b/test/e2e/opencode-live.e2e.ts @@ -8,7 +8,9 @@ * both shapes are asserted against the live service rather than our idea of it. * * Opt-in via `OPENCODE_E2E=1`, and each credentialed block skips when its key is - * absent — so forks and key-less runs stay green instead of failing on a 401. + * absent — so forks and key-less runs stay green instead of failing on a 401 — + * or when `OPENCODE_E2E_KEYED` names a tier below `full` (unset is `full`; CI + * runs `full` nightly and on dispatch). * * OPENCODE_E2E=1 OPENCODE_API_KEY=… OPENCODE_GO_API_KEY=… pnpm run test:e2e * @@ -19,6 +21,7 @@ import { describe, expect, it } from "vitest"; +import { armedKey, keyedTierOf } from "../support/keyed-probes.ts"; import { type GoUsage, OPENCODE_ZEN_CATALOG, @@ -36,8 +39,10 @@ const ZEN_BASE = const GO_BASE = process.env.OPENCODE_GO_BASE_URL ?? "https://opencode.ai/zen/go/v1"; -const ZEN_KEY = process.env.OPENCODE_API_KEY; -const GO_KEY = process.env.OPENCODE_GO_API_KEY; +/** What may spend a key: the keyed blocks here run on the `full` tier only. */ +const TIER = keyedTierOf(process.env.OPENCODE_E2E_KEYED); +const ZEN_KEY = armedKey(TIER, "full", process.env.OPENCODE_API_KEY); +const GO_KEY = armedKey(TIER, "full", process.env.OPENCODE_GO_API_KEY); /** Generous: these cross the public internet from CI. */ const TIMEOUT_MS = 30_000; diff --git a/test/e2e/protocol-routing.e2e.ts b/test/e2e/protocol-routing.e2e.ts index 0abd149..a71d8f3 100644 --- a/test/e2e/protocol-routing.e2e.ts +++ b/test/e2e/protocol-routing.e2e.ts @@ -51,7 +51,12 @@ * for a negative control — the modern `/inference/…` family since P-R1, Go's * Google included. So a model that the gateway serves on the legacy `/zen` * family but refuses on the one the plugin dispatches to turns a case red. - * Probing the legacy family per protocol is P-R2's record. + * The keyed probe of every URL shape per plane and protocol, the legacy family + * included, is `keyed-probes.e2e.ts`. + * + * The keyed cases spend a key only on the `full` tier of `OPENCODE_E2E_KEYED` + * (the default when it is unset; CI runs it nightly and on dispatch), and skip + * otherwise. The keyless cases run on every tier. * * **The last block sends through the plugin's own provider.** The probes above * write their bodies and headers by hand, which is what makes them an exact @@ -85,6 +90,7 @@ import { mountTierA, waitForPlaneOwner, } from "../support/host.ts"; +import { armedKey, keyedTierOf } from "../support/keyed-probes.ts"; import { isModelGoneMessage } from "../support/keyless-matrix.ts"; import { accessDisabled, @@ -114,16 +120,23 @@ import { planeModelFor, } from "../support/src.ts"; import { usesCatalogThinkingLevels } from "../support/thinking-levels.ts"; -import { freeTierClosed, gatewayBusy } from "./verdict.ts"; +import { + freeTierClosed, + freeWalkCappedReason, + freeWalkGap, + gatewayBusy, +} from "./verdict.ts"; const LIVE = process.env.OPENCODE_E2E === "1"; -const ZEN_KEY = process.env.OPENCODE_API_KEY; +/** What may spend a key: the keyed cases here run on the `full` tier only. */ +const TIER = keyedTierOf(process.env.OPENCODE_E2E_KEYED); +const ZEN_KEY = armedKey(TIER, "full", process.env.OPENCODE_API_KEY); /** * The Go plane. A different base URL AND a different credential — which is why * probing its models against the Zen base proves nothing about them. */ -const GO_KEY = process.env.OPENCODE_GO_API_KEY; +const GO_KEY = armedKey(TIER, "full", process.env.OPENCODE_GO_API_KEY); /** A base to probe instead of the plugin's own, per plane (a mirror or a stub). */ const BASE_OVERRIDE: Readonly> = { @@ -824,17 +837,17 @@ describe.skipIf(!LIVE)("live protocol routing", () => { (m) => m.id ); let answered: { model: string; status: number } | undefined; - let busy = false; + const answers: Probe[] = []; for (const model of candidates) { const response = await probe( servedProtocolFor(model) ?? "openai-completions", model ); + answers.push(response); if (gatewayBusy(response)) { // The gateway answered, but about its own load. That is neither a // completion nor a dead tier, and it must not be reported as one. // Asked before the refusals below, which count a 429 among them. - busy = true; continue; } // A gated, withdrawn or refused candidate says nothing about whether @@ -850,12 +863,22 @@ describe.skipIf(!LIVE)("live protocol routing", () => { break; } if (answered === undefined) { - if (busy) { + const gap = freeWalkGap(answers); + if (gap === "busy") { // No candidate answered and at least one was busy, which says // nothing about the tier. context.skip(); return; } + if (gap === "capped") { + // The free tier's usage cap answered for at least one candidate: it + // measures the account this run spends from, not whether the tier + // serves the plugin — the keyless free probe's `free-usage-limited` + // skip, for the same reason. A session gate another candidate met + // is named in the reason, not dropped. + context.skip(true, freeWalkCappedReason(answers)); + return; + } // Reported rather than skipped: the account (or the vendor) closed the // whole free tier, and that is worth a line in the run's output. console.log( diff --git a/test/e2e/verdict.ts b/test/e2e/verdict.ts index 68b7087..50f6fac 100644 --- a/test/e2e/verdict.ts +++ b/test/e2e/verdict.ts @@ -25,6 +25,20 @@ export interface Probe { readonly summary: string; } +/** + * Whether the free tier's usage cap answered: `429 FreeUsageLimitError`, the + * gate refusing the caller's usage of the tier. It is one way the tier is + * closed ({@link freeTierClosed}), but a different fact from the session + * gate: it measures the account a run spends from, not whether the tier + * serves this plugin, so a walk that met the cap reports the cap + * ({@link freeWalkGap}). + * + * @param result - the probe's answer. + * @returns whether the answer names the usage cap. + */ +export const freeUsageCapped = (result: Probe): boolean => + result.summary.includes("FreeUsageLimitError"); + /** * Whether the gateway is refusing the FREE TIER rather than the route. * @@ -41,15 +55,20 @@ export interface Probe { * @returns whether the free tier closed the request. */ export const freeTierClosed = (result: Probe): boolean => - result.summary.includes("FreeTierError"); + result.summary.includes("FreeTierError") || + // The tier's usage cap, `429 FreeUsageLimitError`, is the same gate + // refusing for the caller's usage: closed, not busy (lesson L3). + freeUsageCapped(result); /** * Whether the account cannot spend on this model — an entitlement answer, not a * routing answer. * * Four shapes, and they are not the same refusal: `402` is "no funds", a `403` - * naming `Model access is disabled` is a per-model entitlement, `429` is the - * gateway's own load and says nothing about the route, and `0` is silence. + * naming `Model access is disabled` is a per-model entitlement, a `429` is the + * gateway's own load ({@link gatewayBusy}) or the free tier's usage cap + * ({@link freeUsageCapped}) — neither says anything about the route — and `0` + * is silence. * * @param result - the probe's answer. * @returns whether the account (or the gateway) refused for a reason other than @@ -91,8 +110,57 @@ export const retiredAtTheGateway = (result: Probe): boolean => result.summary.includes("ModelDeprecated") || (result.status === 400 && result.summary.includes("Model is unavailable")); -/** The gateway is busy: it answered, but not about the route. */ -export const gatewayBusy = (result: Probe): boolean => result.status === 429; +/** + * The gateway is busy: it answered, but not about the route. A 429 naming the + * free tier's usage cap is not busy: it is the tier closing + * ({@link freeTierClosed}). + */ +export const gatewayBusy = (result: Probe): boolean => + result.status === 429 && !freeUsageCapped(result); + +/** Why a walk of free candidates ended without a completion. */ +export type FreeWalkGap = "busy" | "capped" | "closed"; + +/** + * Why no free candidate answered with a completion, from every answer the + * walk read: `busy` when the gateway was loaded for any of them + * ({@link gatewayBusy}), which says nothing about the tier; else `capped` + * when any met the usage cap ({@link freeUsageCapped}), which measures the + * account, not the tier; else `closed` — each was gated, withdrawn or + * refused for the account, which is worth reporting rather than skipping. + * + * @param answers - every candidate's answer, in the order sent. + * @returns why the walk found no completion. + */ +export const freeWalkGap = (answers: readonly Probe[]): FreeWalkGap => { + if (answers.some(gatewayBusy)) { + return "busy"; + } + return answers.some(freeUsageCapped) ? "capped" : "closed"; +}; + +/** + * The skip reason of a walk whose gap is `capped` ({@link freeWalkGap}). The + * rule reads ANY capped answer as the cap, so a mixed walk may also have met + * the session gate (`403 FreeTierError`); the reason then says how many + * candidates did, so the gate this case exists to surface is still printed. + * + * @param answers - every candidate's answer, in the order sent. + * @returns the reason the walk skips with. + */ +export const freeWalkCappedReason = (answers: readonly Probe[]): string => { + const reason = + "the free tier's usage cap (429 FreeUsageLimitError) answered, which measures the account, not the tier"; + const gated = answers.filter( + (answer) => + answer.summary.includes("FreeTierError") && !freeUsageCapped(answer) + ).length; + if (gated === 0) { + return reason; + } + const candidates = gated === 1 ? "1 candidate" : `${gated} candidates`; + return `${reason}; ${candidates} also answered 403 FreeTierError (the session gate)`; +}; /** * Whether a candidate model may be USED as a case's subject. diff --git a/test/endpoint-family.test.ts b/test/endpoint-family.test.ts new file mode 100644 index 0000000..a663942 --- /dev/null +++ b/test/endpoint-family.test.ts @@ -0,0 +1,254 @@ +/** + * The endpoint family each `endpoint` setting sends a turn to, pinned (P-R2). + * + * P-R2 measures which families the live gateway serves + * (`test/e2e/keyless-matrix.e2e.ts`, `test/e2e/keyed-probes.e2e.ts`) and keeps + * the default: `auto`, an alias of `modern`. This file holds the two halves of + * that decision offline — the default sends every protocol on both planes to + * the modern family, and `endpoint: legacy` still maps every protocol on both + * planes, Go's Google on its one base. The family `auto` resolves to is one + * constant (`AUTO_ENDPOINT_FAMILY`), but a flip is not a one-word edit: it + * also moves the copy, `TODAY_DISPATCH`, the bases pinned here and the + * re-tagged wire facets (`docs/redirect-planes.md` §13), and this file turns + * red until they move together. `endpoint: legacy` is recorded, not guarded, + * on the live gateway; these pins are what guard its mapping. + * + * @module test/endpoint-family.test + */ +import { getBuiltinModels } from "@earendil-works/pi-ai/providers/all"; +import { describe, expect, it } from "vitest"; + +import { + cellKey, + keylessMatrix, + LEGACY_DISPATCH, + MATRIX_FAMILIES, + matrixUrl, + TODAY_DISPATCH, +} from "./support/keyless-matrix.ts"; +import { + AUTO_ENDPOINT_FAMILY, + baseUrlFor, + CONFIG_DEFAULTS, + DEFAULT_ENDPOINT_FAMILY, + type EndpointChoice, + GO_GOOGLE_BASE_URL, + LEGACY_BASE, + OPENCODE_FAMILY_API, + OPENCODE_GO_CATALOG, + OPENCODE_ZEN_CATALOG, + PLANE_ROUTES, + type PlanePiModel, + planeModelFor, + type PlaneRoute, + type Protocol, + PROTOCOLS, + resolveConfig, +} from "./support/src.ts"; + +const familyApiOf = (plane: PlaneRoute): Readonly> => + OPENCODE_FAMILY_API[plane === "opencode" ? "zen" : "go"]; + +/** + * The path each protocol appends to a model's base, restated from + * `docs/redirect-planes.md` §9 (the query aside): the Anthropic SDK appends + * its own `/v1/messages`, and Google names the model in the path. + */ +const SENT_PATH: Readonly string>> = { + "anthropic-messages": () => "/v1/messages", + "google-generative-ai": (id) => `/models/${id}:streamGenerateContent`, + "openai-completions": () => "/chat/completions", + "openai-responses": () => "/responses", +}; + +/** + * The route's defaults as pi-ai's `Config` fills them. Stand-in values, on + * purpose not either channel's: the plugin restates none of pi-ai's defaults + * (D17), and the family a turn is sent to does not depend on them. + */ +const DEFAULTS = { contextWindow: 123_456, input: ["text"], maxTokens: 6_543 }; + +/** Every plane × protocol's base under one `endpoint` setting. */ +const basesFor = (endpoint: EndpointChoice): Record => + Object.fromEntries( + PLANE_ROUTES.flatMap((plane) => + PROTOCOLS.map((protocol) => [ + `${plane} ${protocol}`, + baseUrlFor(plane, protocol, undefined, endpoint, familyApiOf(plane)), + ]) + ) + ); + +describe("endpoint family · the default", () => { + it("the default is auto, and auto sends every protocol on both planes to the modern family", () => { + expect(DEFAULT_ENDPOINT_FAMILY).toBe("auto"); + expect(CONFIG_DEFAULTS.endpoint).toBe("auto"); + expect(resolveConfig({}).endpoint).toBe("auto"); + // A family the setting does not know degrades to the default. + expect(resolveConfig({ endpoint: "zen" as never }).endpoint).toBe("auto"); + + const bases = basesFor(DEFAULT_ENDPOINT_FAMILY); + expect(bases).toEqual({ + "opencode anthropic-messages": "https://opencode.ai/inference/anthropic", + "opencode google-generative-ai": + "https://opencode.ai/inference/google/v1beta", + "opencode openai-completions": "https://opencode.ai/inference/openai/v1", + "opencode openai-responses": "https://opencode.ai/inference/openai/v1", + "opencode-go anthropic-messages": + "https://opencode.ai/inference/go/anthropic", + "opencode-go google-generative-ai": + "https://opencode.ai/inference/go/google/v1beta", + "opencode-go openai-completions": + "https://opencode.ai/inference/go/openai/v1", + "opencode-go openai-responses": + "https://opencode.ai/inference/go/openai/v1", + }); + const modern = basesFor("modern"); + for (const [cell, base] of Object.entries(bases)) { + expect(base, cell).toBe(modern[cell]); + } + }); +}); + +describe("endpoint family · the one constant", () => { + it("snapshots AUTO_ENDPOINT_FAMILY as modern; auto's bases equal the constant's, a guard that bites once the constant flips", () => { + // The family `auto` resolves to; P-R2 keeps it while the families are + // measured in CI (docs/redirect-planes.md §13, pending the first run), and + // a measured flip is P-R2b's `fix:`. + expect(AUTO_ENDPOINT_FAMILY).toBe("modern"); + // While the constant is `modern` this loop restates the first describe. + // It is the guard for the flip: change the constant, and a `baseUrlFor` + // that ignored it would leave `auto` on the old family and fail here. + for (const plane of PLANE_ROUTES) { + for (const protocol of PROTOCOLS) { + expect( + baseUrlFor(plane, protocol, undefined, "auto", familyApiOf(plane)), + `${plane} ${protocol}` + ).toBe( + baseUrlFor( + plane, + protocol, + undefined, + AUTO_ENDPOINT_FAMILY, + familyApiOf(plane) + ) + ); + } + } + expect( + [...TODAY_DISPATCH].every((cell) => + cell.endsWith(` ${AUTO_ENDPOINT_FAMILY}`) + ) + ).toBe(true); + }); +}); + +describe("endpoint family · legacy", () => { + it("endpoint: legacy maps every protocol on both planes: the /zen bases, Go's Google on its one base", () => { + const bases = basesFor("legacy"); + expect(bases).toEqual({ + "opencode anthropic-messages": "https://opencode.ai/zen", + "opencode google-generative-ai": "https://opencode.ai/zen/v1", + "opencode openai-completions": "https://opencode.ai/zen/v1", + "opencode openai-responses": "https://opencode.ai/zen/v1", + "opencode-go anthropic-messages": "https://opencode.ai/zen/go", + "opencode-go google-generative-ai": + "https://opencode.ai/inference/go/google/v1beta", + "opencode-go openai-completions": "https://opencode.ai/zen/go/v1", + "opencode-go openai-responses": "https://opencode.ai/zen/go/v1", + }); + expect(Object.keys(bases)).toHaveLength(8); + for (const plane of PLANE_ROUTES) { + for (const protocol of PROTOCOLS) { + const base = bases[`${plane} ${protocol}`]; + expect(base, `${plane} ${protocol}`).toBeDefined(); + expect(base).toBe(LEGACY_BASE[plane][protocol]); + const url = new URL(base ?? ""); + expect(url.hostname).toBe("opencode.ai"); + if (plane === "opencode-go" && protocol === "google-generative-ai") { + expect(base).toBe(GO_GOOGLE_BASE_URL); + } else { + expect(url.pathname.startsWith("/zen"), `${plane} ${protocol}`).toBe( + true + ); + } + } + } + }); +}); + +describe("endpoint family · what a turn reaches", () => { + const cells = keylessMatrix({ + go: OPENCODE_GO_CATALOG, + zen: OPENCODE_ZEN_CATALOG, + }); + + /** The matrix cells a turn of each plane × protocol reaches under `endpoint`. */ + const reachedUnder = (endpoint: EndpointChoice): string[] => { + const reached: string[] = []; + for (const plane of PLANE_ROUTES) { + const catalog = + plane === "opencode" ? OPENCODE_ZEN_CATALOG : OPENCODE_GO_CATALOG; + const pis = getBuiltinModels(plane as never) as unknown as PlanePiModel[]; + for (const protocol of PROTOCOLS) { + const id = cells.find( + (cell) => + cell.plane === plane && + cell.protocol === protocol && + cell.family === "modern" + )?.model; + expect(id, `${plane} ${protocol}`).toBeDefined(); + const spec = catalog.find((row) => row.id === id); + const pi = pis.find((row) => row.id === id); + const result = planeModelFor({ + defaults: DEFAULTS, + endpoint, + entry: { id }, + familyApi: familyApiOf(plane), + route: plane, + ...(spec === undefined ? {} : { spec }), + ...(pi === undefined ? {} : { pi }), + }); + expect(result.error, `${plane} ${protocol}`).toBeUndefined(); + expect(result.model?.api).toBe(protocol); + const sent = + new URL(result.model?.baseUrl ?? "").pathname.replace(/\/$/u, "") + + SENT_PATH[protocol](id ?? ""); + const matches = MATRIX_FAMILIES.filter( + (family) => + new URL(matrixUrl(plane, protocol, family, id)).pathname === sent + ).map((family) => cellKey({ family, plane, protocol })); + expect( + matches, + `${endpoint} ${plane} ${protocol} ${sent}` + ).toHaveLength(1); + reached.push(matches[0] ?? ""); + } + } + return reached; + }; + + it("a turn reaches exactly the cells each setting dispatches to: auto and modern TODAY_DISPATCH, legacy LEGACY_DISPATCH", () => { + const expected: Record> = { + auto: TODAY_DISPATCH, + legacy: LEGACY_DISPATCH, + modern: TODAY_DISPATCH, + }; + for (const endpoint of [ + DEFAULT_ENDPOINT_FAMILY, + "modern", + "legacy", + ] as const) { + const reached = reachedUnder(endpoint); + expect(reached.toSorted(), endpoint).toEqual( + [...expected[endpoint]].toSorted() + ); + expect(new Set(reached).size, endpoint).toBe(8); + // Every plane × protocol once. + expect( + new Set(reached.map((cell) => cell.split(" ").slice(0, 2).join(" "))) + .size + ).toBe(8); + } + }); +}); diff --git a/test/endpoint-record.test.ts b/test/endpoint-record.test.ts new file mode 100644 index 0000000..4eb06e8 --- /dev/null +++ b/test/endpoint-record.test.ts @@ -0,0 +1,385 @@ +/** + * The dated record the live suites write, and the doc section that holds it. + * + * `test/e2e/keyless-matrix.e2e.ts` and `test/e2e/keyed-probes.e2e.ts` print + * one markdown table each into the CI run summary and the `endpoint-families` + * artifact; a run's tables are pasted into `docs/redirect-planes.md` §13 with + * their date. This file pins the formatter line by line, and holds the doc's + * tables to the formatter's columns and row order, so a pasted run and the + * placeholder it replaces have the same shape. It compares trimmed cells, so a + * formatter re-padding the tables cannot break it, and it never asserts what a + * cell SAYS, so pasting real results keeps it green. + * + * @module test/endpoint-record.test + */ +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +import { + freeProbeLine, + RECORD_HEADER, + RECORD_ROWS, + RECORD_RULE, + type RecordedProbe, + recordCell, + recordTable, + retryAfterLine, + runReference, + systemOneLine, + VARIANT_HEADER, + VARIANT_ROWS, + VARIANT_RULE, + variantTable, +} from "./support/endpoint-record.ts"; +import { KEYED_VARIANTS } from "./support/keyed-probes.ts"; +import { MATRIX_FAMILIES, MATRIX_PROTOCOLS } from "./support/keyless-matrix.ts"; + +/** One probe per cell but one: root answers 404, the rest 401, Go's Google has no model. */ +const PROBES: RecordedProbe[] = (["opencode", "opencode-go"] as const) + .flatMap((plane) => + MATRIX_PROTOCOLS.flatMap((protocol) => + MATRIX_FAMILIES.map((family): RecordedProbe => { + if (plane === "opencode-go" && protocol === "google-generative-ai") { + return { + family, + model: undefined, + plane, + protocol, + status: undefined, + verdict: "no-model", + }; + } + return { + family, + model: `m-${protocol}`, + plane, + protocol, + status: family === "root" ? 404 : 401, + verdict: family === "root" ? "path-gone" : "path-exists", + }; + }) + ) + ) + .filter( + (probe) => + !( + probe.plane === "opencode" && + probe.protocol === "openai-completions" && + probe.family === "root" + ) + ); + +const cellsOf = (line: string): string[] => + line + .split("|") + .slice(1, -1) + .map((cell) => cell.trim()); + +describe("endpoint record · the table", () => { + it("prints one dated row per plane × protocol, a cell per shape with verdict, status and dispatch marks", () => { + expect(PROBES).toHaveLength(23); + expect( + recordTable({ + date: "2026-10-24", + probes: PROBES, + run: "a local run", + title: "Keyless", + }).split("\n") + ).toEqual([ + "### Keyless · measured 2026-10-24 (UTC) · a local run", + "", + RECORD_HEADER, + RECORD_RULE, + "| `opencode` | `openai-completions` | `m-openai-completions` | path-exists (401) † | not sent | path-exists (401) * |", + "| `opencode` | `openai-responses` | `m-openai-responses` | path-exists (401) † | path-gone (404) | path-exists (401) * |", + "| `opencode` | `anthropic-messages` | `m-anthropic-messages` | path-exists (401) † | path-gone (404) | path-exists (401) * |", + "| `opencode` | `google-generative-ai` | `m-google-generative-ai` | path-exists (401) † | path-gone (404) | path-exists (401) * |", + "| `opencode-go` | `openai-completions` | `m-openai-completions` | path-exists (401) † | path-gone (404) | path-exists (401) * |", + "| `opencode-go` | `openai-responses` | `m-openai-responses` | path-exists (401) † | path-gone (404) | path-exists (401) * |", + "| `opencode-go` | `anthropic-messages` | `m-anthropic-messages` | path-exists (401) † | path-gone (404) | path-exists (401) * |", + "| `opencode-go` | `google-generative-ai` | — | no-model | no-model | no-model * † |", + ]); + expect(RECORD_HEADER).toBe( + "| Plane | Protocol | Model | `/zen(/go)/v1` (legacy) | `/inference(/go)/v1` (root) | `/inference(/go)/{openai,anthropic,google}/…` (modern) |" + ); + expect(RECORD_RULE).toBe("| :-- | :-- | :-- | :-- | :-- | :-- |"); + }); + + it("names each cell, its status and its marks", () => { + expect(recordCell("opencode openai-completions root")).toBe("not sent"); + expect( + recordCell("opencode openai-completions modern", { + family: "modern", + model: "m", + plane: "opencode", + protocol: "openai-completions", + status: 200, + verdict: "served", + }) + ).toBe("served (200) *"); + expect( + recordCell("opencode-go openai-responses legacy", { + family: "legacy", + model: "m", + plane: "opencode-go", + protocol: "openai-responses", + status: undefined, + verdict: "transport-error", + }) + ).toBe("transport-error †"); + }); + + it("names the path a redirect pointed at, with a `|` encoded so the row stays six cells", () => { + const redirected = (location: string): RecordedProbe => ({ + family: "legacy", + location, + model: "m", + plane: "opencode", + protocol: "openai-completions", + status: 308, + verdict: "redirected", + }); + expect( + recordCell( + "opencode openai-completions legacy", + redirected("/inference/openai/v1/chat/completions") + ) + ).toBe("redirected (308 → /inference/openai/v1/chat/completions) †"); + const piped = recordCell( + "opencode openai-completions legacy", + redirected("/a|b") + ); + expect(piped).toBe("redirected (308 → /a%7Cb) †"); + const table = recordTable({ + date: "2026-10-24", + probes: [redirected("/a|b")], + run: "a local run", + title: "Keyed", + }).split("\n"); + expect(cellsOf(table[4] ?? "")).toHaveLength(6); + expect(table[4]).toContain("redirected (308 → /a%7Cb) †"); + // A redirect with no status kept still names its target. + expect( + recordCell("opencode openai-completions root", { + ...redirected("/x"), + family: "root", + status: undefined, + }) + ).toBe("redirected → /x"); + }); + + it("orders the rows plane by plane, in the matrix's protocol order", () => { + expect(RECORD_ROWS).toEqual([ + ...MATRIX_PROTOCOLS.map((protocol) => ["opencode", protocol]), + ...MATRIX_PROTOCOLS.map((protocol) => ["opencode-go", protocol]), + ]); + expect(RECORD_ROWS).toHaveLength(8); + }); + + it("links the run that measured it, or says it was local", () => { + expect( + runReference({ + repository: "o/r", + runId: "42", + server: "https://github.com", + }) + ).toBe("[run 42](https://github.com/o/r/actions/runs/42)"); + for (const parts of [ + { repository: undefined, runId: "42", server: "https://github.com" }, + { repository: "", runId: "42", server: "https://github.com" }, + { repository: "o/r", runId: undefined, server: "https://github.com" }, + { repository: "o/r", runId: "", server: "https://github.com" }, + { repository: "o/r", runId: "42", server: undefined }, + { repository: "o/r", runId: "42", server: "" }, + ]) { + expect(runReference(parts)).toBe("a local run"); + } + }); + + it("prints the free probe on one line", () => { + expect( + freeProbeLine({ model: "beam-free", status: 200, verdict: "served" }) + ).toBe( + "Free model on `/inference/openai/v1/chat/completions`: `beam-free` → served (200)." + ); + expect( + freeProbeLine({ + model: undefined, + status: undefined, + verdict: "no-model", + }) + ).toBe( + "Free model on `/inference/openai/v1/chat/completions`: no model → no-model." + ); + }); +}); + +describe("endpoint record · the records beside the tables (L4, L5, L9, L10, L13)", () => { + it("prints the System One family on one line, a shape it did not reach as not sent", () => { + expect( + systemOneLine([ + { family: "legacy", status: 401, verdict: "path-exists" }, + { family: "modern", status: 404, verdict: "path-gone" }, + ]) + ).toBe( + "System One on the Zen plane, keyless (nothing dispatches there): `/zen/v1/systemone` (legacy) → path-exists (401); `/inference/v1/systemone` (root) → not sent; `/inference/systemone/v1/systemone` (modern) → path-gone (404)." + ); + expect( + systemOneLine([ + { family: "root", status: undefined, verdict: "transport-error" }, + ]) + ).toContain("`/inference/v1/systemone` (root) → transport-error;"); + }); + + it("prints one dated variants row per plane and variant, in order, a variant not sent as such", () => { + const lines = variantTable({ + date: "2026-11-04", + probes: [ + { + model: "m-chat", + plane: "opencode", + status: 200, + variant: "baseline", + verdict: "served", + }, + { + model: "m-chat", + plane: "opencode", + status: 400, + variant: "request-id", + verdict: "path-exists", + }, + { + model: undefined, + plane: "opencode-go", + status: undefined, + variant: "reasoning-none", + verdict: "no-model", + }, + ], + run: "a local run", + }).split("\n"); + expect(lines.slice(0, 4)).toEqual([ + "### Keyed variants · measured 2026-11-04 (UTC) · a local run", + "", + VARIANT_HEADER, + VARIANT_RULE, + ]); + expect(lines).toHaveLength(4 + 2 * KEYED_VARIANTS.length); + expect(lines[4]).toBe( + "| `opencode` | the probe identity: no `x-opencode-request`, `x-opencode-project: global` | L4, L5 | `m-chat` | served (200) |" + ); + expect(lines[5]).toBe( + "| `opencode` | `x-opencode-request: msg_…` | L4 | `m-chat` | path-exists (400) |" + ); + expect(lines[6]).toBe( + "| `opencode` | `x-opencode-project: ` | L5 | — | not sent |" + ); + expect(lines.at(-1)).toBe( + '| `opencode-go` | `reasoning_effort: "none"` | L10 | — | no-model |' + ); + expect( + VARIANT_ROWS.map(([plane, variant]) => `${plane} ${variant.id}`) + ).toEqual( + (["opencode", "opencode-go"] as const).flatMap((plane) => + KEYED_VARIANTS.map((variant) => `${plane} ${variant.id}`) + ) + ); + }); + + it("prints every keyed 429's Retry-After, absent included, with nothing that could break the line", () => { + expect(retryAfterLine([])).toBe( + "`Retry-After` on a keyed 429: no 429 was answered." + ); + expect( + retryAfterLine([ + { cell: "opencode-go openai-completions modern", value: "30" }, + { cell: "opencode baseline", value: null }, + { + cell: "opencode-go request-id", + value: "Wed, 21 Oct 2026 07:28:00 GMT", + }, + { cell: "opencode root", value: "1`|\n2" }, + ]) + ).toBe( + "`Retry-After` on a keyed 429: opencode-go openai-completions modern → `30`; opencode baseline → absent; opencode-go request-id → `Wed, 21 Oct 2026 07:28:00 GMT`; opencode root → `1b2`." + ); + }); +}); + +describe("endpoint record · docs/redirect-planes.md §13", () => { + it("docs/redirect-planes.md §13 holds the keyless and keyed tables in the formatter's columns and row order, and the free-probe line", () => { + const doc = readFileSync( + new URL("../docs/redirect-planes.md", import.meta.url), + "utf8" + ); + const start = doc.indexOf("\n## 13. "); + expect(start).toBeGreaterThan(-1); + const section = doc.slice(start); + const lines = section.split("\n"); + const header = cellsOf(RECORD_HEADER); + const headerAt = lines.flatMap((line, index) => + line.startsWith("|") && + JSON.stringify(cellsOf(line)) === JSON.stringify(header) + ? [index] + : [] + ); + expect(headerAt).toHaveLength(2); + for (const at of headerAt) { + expect(lines[at + 1]).toMatch(/^\|(\s*:?-+:?\s*\|){6}$/u); + const rows = lines.slice(at + 2, at + 10); + expect(rows.map((row) => cellsOf(row).slice(0, 2))).toEqual( + RECORD_ROWS.map(([plane, protocol]) => [ + `\`${plane}\``, + `\`${protocol}\``, + ]) + ); + for (const row of rows) { + expect(cellsOf(row), row).toHaveLength(6); + } + } + const headings = lines.filter((line) => line.startsWith("### ")); + expect(headings).toHaveLength(3); + expect(headings[0]).toMatch( + /^### Keyless · (pending the first CI run|measured \d{4}-\d{2}-\d{2} \(UTC\) · .+)$/u + ); + expect(headings[1]).toMatch( + /^### Keyed · (pending the first CI run|measured \d{4}-\d{2}-\d{2} \(UTC\) · .+)$/u + ); + expect(section).toMatch( + /^Free model on `\/inference\/openai\/v1\/chat\/completions`: .+$/mu + ); + expect(section).toMatch( + /^System One on the Zen plane, keyless \(nothing dispatches there\): .+$/mu + ); + expect(section).toMatch(/^`Retry-After` on a keyed 429: .+$/mu); + }); + + it("holds the keyed variants table in the formatter's columns, labels and row order", () => { + const doc = readFileSync( + new URL("../docs/redirect-planes.md", import.meta.url), + "utf8" + ); + const lines = doc.slice(doc.indexOf("\n## 13. ")).split("\n"); + const heading = lines.findIndex((line) => + line.startsWith("### Keyed variants · ") + ); + expect(lines[heading]).toMatch( + /^### Keyed variants · (pending the first CI run|measured \d{4}-\d{2}-\d{2} \(UTC\) · .+)$/u + ); + expect(cellsOf(lines[heading + 2] ?? "")).toEqual(cellsOf(VARIANT_HEADER)); + expect(lines[heading + 3]).toMatch(/^\|(\s*:?-+:?\s*\|){5}$/u); + const rows = lines.slice(heading + 4, heading + 4 + VARIANT_ROWS.length); + expect(rows.map((row) => cellsOf(row).slice(0, 3))).toEqual( + VARIANT_ROWS.map(([plane, variant]) => [ + `\`${plane}\``, + variant.label, + variant.lesson, + ]) + ); + for (const row of rows) { + expect(cellsOf(row), row).toHaveLength(5); + } + }); +}); diff --git a/test/keyed-probes.test.ts b/test/keyed-probes.test.ts new file mode 100644 index 0000000..3425aa7 --- /dev/null +++ b/test/keyed-probes.test.ts @@ -0,0 +1,1514 @@ +/** + * The keyed probes' tier, cells and classifier, offline. + * + * The live half (`test/e2e/keyed-probes.e2e.ts`) sends one request per plane × + * protocol × URL shape with the plane's own key, which only CI can do; every + * decision it makes is made here — which tier arms what, which headers carry + * the key and the plugin's identity, how small each body is, how an answer + * (and a retried one) is read, what a plane's verdicts decide, that a key + * never reaches a printed excerpt, which pull requests CI arms it on, how the + * record is kept, and — read from the live files — that every live suite + * reads its key at its own tier and no probe follows a redirect. The + * keyed cells also go through the offline gateway mock's own judge + * (`handleGatewayRequest`), which refuses a request with no session header as + * the live Go plane does. + * + * @module test/keyed-probes.test + */ +import { AsyncLocalStorage } from "node:async_hooks"; +import { readdirSync, readFileSync } from "node:fs"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { handleGatewayRequest } from "./mock/opencode-gateway.ts"; +import { + armedKey, + breaksDefault, + classifyKeyedProbe, + credentialHeaders, + excerpt, + FLIP_EVIDENCE, + isArmed, + KEYED_VARIANTS, + keyedBody, + keyedCell, + keyedTierOf, + keyedVariantCells, + keyedVerdictOf, + planeOutcome, + pluginIdentityHeaders, + PROBE_IDENTITY, + PROBE_PROJECTS, + PROBE_REQUEST_ID, + PROBE_SESSION_ID, + reasoningCompletionsModel, + retryAfterOf, +} from "./support/keyed-probes.ts"; +import { + cellKey, + keylessMatrix, + type MatrixCell, + type MatrixPlane, + TODAY_DISPATCH, +} from "./support/keyless-matrix.ts"; +import { + type ActiveTurnState, + authHeaderFor, + type CatalogModelSpec, + clearCapturedApiKeys, + CONFIG_DEFAULTS, + isOpenCodeMessageId, + isOpenCodeSessionId, + newOpenCodeMessageId, + OPENCODE_GO_CATALOG, + OPENCODE_PATCH_USER_AGENT, + OPENCODE_UA, + OPENCODE_ZEN_CATALOG, + patchFetch, + resolveConfig, + SESSION_HEADER, +} from "./support/src.ts"; +import { blankCommentLines } from "./support/strip-comments.ts"; + +const spec = (id: string, provider_npm?: string): CatalogModelSpec => ({ + context_window: 1000, + id, + input_modalities: ["text"], + max_output_tokens: 100, + name: id, + ...(provider_npm === undefined ? {} : { provider_npm }), +}); + +/** The credential headers, lower-cased: a keyless cell carries none of them. */ +const CREDENTIAL_HEADERS = new Set([ + "authorization", + "x-api-key", + "x-goog-api-key", +]); + +/** A live suite's source with its comment lines blanked, as the gates read it. */ +const liveSource = (name: string): string => + blankCommentLines( + readFileSync(new URL(`e2e/${name}`, import.meta.url), "utf8") + ); + +/** One paid model per protocol on each plane. */ +const ZEN = [ + spec("zen-chat"), + spec("zen-resp", "@ai-sdk/openai"), + spec("zen-claude", "@ai-sdk/anthropic"), + spec("zen-gemini", "@ai-sdk/google"), +]; +const GO = [ + spec("go-chat"), + spec("go-resp", "@ai-sdk/openai"), + spec("go-claude", "@ai-sdk/anthropic"), + spec("go-gemini", "@ai-sdk/google"), +]; + +// The fetch-patch case below captures the key it sends, process-wide. +afterEach(() => { + clearCapturedApiKeys(); +}); + +describe("keyed probes · the tier", () => { + it.each([ + [undefined, "full"], + ["", "full"], + [" ", "full"], + ["none", "none"], + ["probes", "probes"], + [" probes ", "probes"], + ["full", "full"], + ] as const)("OPENCODE_E2E_KEYED=%j → %s", (value, tier) => { + expect(keyedTierOf(value)).toBe(tier); + }); + + it("the tier: unset is full, a known value is itself, anything else fails the run; arming", () => { + expect(() => keyedTierOf("prob")).toThrow( + 'OPENCODE_E2E_KEYED must be one of none, probes, full; got "prob"' + ); + expect(isArmed("none", "probes")).toBe(false); + expect(isArmed("none", "full")).toBe(false); + expect(isArmed("probes", "probes")).toBe(true); + expect(isArmed("probes", "full")).toBe(false); + expect(isArmed("full", "probes")).toBe(true); + expect(isArmed("full", "full")).toBe(true); + + expect(armedKey("full", "full", "k")).toBe("k"); + expect(armedKey("probes", "probes", "k")).toBe("k"); + expect(armedKey("probes", "full", "k")).toBeUndefined(); + expect(armedKey("full", "full", "")).toBeUndefined(); + expect(armedKey("full", "full")).toBeUndefined(); + expect(armedKey("none", "probes", "k")).toBeUndefined(); + }); +}); + +describe("keyed probes · the cells", () => { + it("a keyed cell is the keyless cell with the plugin's identity, the plane's key in the header its protocol reads, and a tiny body each endpoint accepts with a key", () => { + const cells = keylessMatrix({ go: GO, zen: ZEN }); + expect(cells).toHaveLength(24); + for (const cell of cells) { + expect(cell.model, cell.url).toBeDefined(); + const keyed = keyedCell(cell, "test-key"); + expect(keyed.url).toBe(cell.url); + expect(keyed.model).toBe(cell.model); + expect(keyed.family).toBe(cell.family); + expect(keyed.plane).toBe(cell.plane); + expect(keyed.protocol).toBe(cell.protocol); + expect(keyed.method).toBe(cell.method); + // The probe and the plugin's fetch patch agree on every URL, the root + // and legacy shapes included. + expect( + Object.keys(credentialHeaders(cell.protocol, "test-key")), + cell.url + ).toEqual([authHeaderFor(cell.url)]); + expect(keyed.headers["content-type"]).toBe("application/json"); + // The plugin's identity, on every cell: a turn's user agent (not the + // one the plugin signs its own requests with), the origin client, the + // project, and one in-band session id under both headers. + expect(keyed.headers["user-agent"]).toBe(OPENCODE_UA); + expect(keyed.headers["user-agent"]).not.toBe(OPENCODE_PATCH_USER_AGENT); + expect(keyed.headers["x-opencode-client"]).toBe( + CONFIG_DEFAULTS.originClient + ); + expect(keyed.headers["x-opencode-project"]).toBe("global"); + expect(keyed.headers[SESSION_HEADER]).toBe(PROBE_SESSION_ID); + expect(keyed.headers["x-opencode-session-id"]).toBe(PROBE_SESSION_ID); + if ( + cell.protocol === "openai-completions" || + cell.protocol === "openai-responses" + ) { + expect(keyed.headers.authorization).toBe("Bearer test-key"); + } else if (cell.protocol === "anthropic-messages") { + expect(keyed.headers["x-api-key"]).toBe("test-key"); + expect(keyed.headers["anthropic-version"]).toBe("2023-06-01"); + } else { + expect(keyed.headers["x-goog-api-key"]).toBe("test-key"); + } + expect(keyed.body).toEqual(keyedBody(cell.protocol, cell.model ?? "")); + } + // The keyless cells themselves are untouched: no credential header on any. + for (const cell of cells) { + expect( + Object.keys(cell.headers).filter((header) => + CREDENTIAL_HEADERS.has(header.toLowerCase()) + ), + cell.url + ).toEqual([]); + } + + expect(keyedBody("openai-completions", "m")).toEqual({ + max_tokens: 8, + messages: [{ content: "hi", role: "user" }], + model: "m", + stream: false, + }); + expect(keyedBody("openai-responses", "m")).toEqual({ + input: "hi", + max_output_tokens: 16, + model: "m", + }); + expect(keyedBody("anthropic-messages", "m")).toEqual({ + max_tokens: 1, + messages: [{ content: "hi", role: "user" }], + model: "m", + }); + expect(keyedBody("google-generative-ai", "m")).toEqual({ + contents: [{ parts: [{ text: "hi" }], role: "user" }], + generationConfig: { maxOutputTokens: 8 }, + }); + + const modelless = keylessMatrix({ go: [], zen: ZEN }).find( + (cell) => cell.model === undefined + ); + expect(modelless).toBeDefined(); + if (modelless !== undefined) { + expect(keyedCell(modelless, "test-key").body).toBeUndefined(); + } + }); + + it.for(["opencode", "opencode-go"] as const)( + "%s: every identity header on a keyed cell is what the real fetch patch puts on a turn, but the per-request message id it leaves out on purpose", + async (plane) => { + // Derived, not restated: one turn through the real `patchFetch`, with + // the defaults, the probe's session id and a message id as a real + // turn's state carries (`stream-hook.ts`), over a recording fetch. A + // change to the patch's identity — its user agent, a header added or + // renamed — or to `PROBE_IDENTITY`, the constant the live suite sends, + // fails here rather than leaving the live probe behind. + const cell = keylessMatrix({ go: GO, zen: ZEN }).find( + (candidate) => + candidate.plane === plane && + candidate.family === "modern" && + candidate.protocol === "openai-completions" + ); + expect(cell).toBeDefined(); + if (cell === undefined) { + return; + } + let wire = new Headers(); + const recording = ( + _input: RequestInfo | URL, + init?: RequestInit + ): Promise => { + wire = new Headers(init?.headers); + return Promise.resolve(new Response("{}")); + }; + const als = new AsyncLocalStorage(); + const patched = patchFetch( + recording as typeof fetch, + als, + resolveConfig() + ); + const requestValue = newOpenCodeMessageId(); + await als.run( + { provider: plane, requestValue, value: PROBE_SESSION_ID }, + () => + patched(cell.url, { + body: JSON.stringify(keyedBody(cell.protocol, cell.model ?? "m")), + headers: { + authorization: "Bearer sk-probe-0123456789abcdef", + "content-type": "application/json", + }, + method: "POST", + }) + ); + // The one exclusion, by name: the turn's message id, which the gateway + // reads only as a metric and which is fresh per request, so the probe + // identity carries none (`pluginIdentityHeaders`). + const METRIC_ONLY = "x-opencode-request"; + expect(wire.get(METRIC_ONLY)).toBe(requestValue); + expect(isOpenCodeMessageId(requestValue)).toBe(true); + expect(PROBE_IDENTITY).not.toHaveProperty(METRIC_ONLY); + const identityOnWire = [...wire.keys()].filter( + (name) => + name !== METRIC_ONLY && + (name === "user-agent" || name.startsWith("x-opencode-")) + ); + expect(identityOnWire.toSorted()).toEqual( + Object.keys(PROBE_IDENTITY).toSorted() + ); + const keyed = keyedCell(cell, "sk-probe-0123456789abcdef"); + for (const name of identityOnWire) { + expect(keyed.headers[name], name).toBe(wire.get(name)); + } + expect(wire.get("user-agent")).toBe(OPENCODE_UA); + } + ); + + it("the live suite sends PROBE_IDENTITY and has no identity of its own", () => { + // The live file is not loaded offline, so its wiring is read: every + // keyed cell it builds takes keyedCell's default identity, the constant + // the case above holds to the real fetch patch. + const live = blankCommentLines( + readFileSync(new URL("e2e/keyed-probes.e2e.ts", import.meta.url), "utf8") + ); + const calls = [...live.matchAll(/keyedCell\(([^)]*)\)/gu)].map( + (match) => match[1] ?? "" + ); + expect(calls.length).toBeGreaterThan(0); + for (const args of calls) { + expect(args.split(",").map((arg) => arg.trim())).toHaveLength(2); + } + for (const name of [ + "pluginIdentityHeaders", + "OPENCODE_UA", + "OPENCODE_PATCH_USER_AGENT", + '"user-agent"', + ]) { + expect(live, name).not.toContain(name); + } + expect(pluginIdentityHeaders(OPENCODE_UA)).toEqual(PROBE_IDENTITY); + }); + + it("every live suite reads a key only through armedKey at its own tier: the keyed probes at `probes`, the full keyed suites at `full`, the keyless matrix never", () => { + // The live files are not loaded offline, and ci.yml passes both secrets + // on a `probes` pull request, so a full suite that read a key straight + // from the environment would spend it on every such pull request, and + // keyed probes armed at `full` would never probe one: both stay green + // everywhere else. + const NEED: Readonly> = { + "dispatch-levels.e2e.ts": "full", + "keyed-probes.e2e.ts": "probes", + "opencode-live.e2e.ts": "full", + "protocol-routing.e2e.ts": "full", + }; + const files = readdirSync(new URL("e2e/", import.meta.url)).filter((name) => + name.endsWith(".e2e.ts") + ); + // The input was read: the four keyed suites and the keyless matrix are + // among the files walked. + for (const name of [...Object.keys(NEED), "keyless-matrix.e2e.ts"]) { + expect(files, name).toContain(name); + } + const READ = /process\.env(?:\.|\[\s*["'`])OPENCODE_(?:GO_)?API_KEY\b/gu; + const ARMED = + /armedKey\(\s*TIER,\s*"(full|probes)",\s*process\.env\.OPENCODE_(?:GO_)?API_KEY\s*\)/gu; + for (const name of files) { + const live = liveSource(name); + const reads = [...live.matchAll(READ)].length; + const armed = [...live.matchAll(ARMED)].map((match) => match[1]); + const need = NEED[name]; + expect(live, `${name} destructures a key out of process.env`).not.toMatch( + /\}\s*=\s*process\.env\b/u + ); + if (need === undefined) { + expect(reads, `${name} reads a key`).toBe(0); + continue; + } + // Both planes' keys, each through armedKey at the suite's tier, and no + // other read of either. + expect(reads, name).toBe(2); + expect(armed, name).toEqual([need, need]); + expect(live, name).toContain( + "const TIER = keyedTierOf(process.env.OPENCODE_E2E_KEYED);" + ); + } + expect(liveSource("keyless-matrix.e2e.ts")).not.toContain( + "OPENCODE_E2E_KEYED" + ); + }); + + it('no probe follows a redirect: every request the two probe suites send is `redirect: "manual"`, but the catalog GET, which carries no key', () => { + // A followed redirect never shows its 3xx, so `redirected` could not + // occur live, a moved path would be judged by its target's answer, and a + // cross-origin hop would carry `x-api-key` / `x-goog-api-key` with it. + for (const name of ["keyless-matrix.e2e.ts", "keyed-probes.e2e.ts"]) { + const calls = [ + ...liveSource(name).matchAll(/\bfetch\(\s*([^,]+),\s*\{([^}]*)\}/gu), + ].map((match) => ({ options: match[2] ?? "", target: match[1] ?? "" })); + const probes = calls.filter( + (call) => call.target.trim() !== "MODELS_OPENCODE_URL" + ); + // The input was read: one catalog GET and at least one probe each. + expect(calls.length - probes.length, name).toBe(1); + expect(probes.length, name).toBeGreaterThan(0); + for (const call of probes) { + expect(call.options, `${name}: fetch(${call.target})`).toMatch( + /\bredirect:\s*"manual"/u + ); + } + // And no fetch call escaped the pattern above (one without an inline + // options object). + expect([...liveSource(name).matchAll(/\bfetch\(/gu)].length, name).toBe( + calls.length + ); + } + }); + + it("the probe's session id is the client's shape, inside the band the gateway admits", () => { + expect(isOpenCodeSessionId(PROBE_SESSION_ID)).toBe(true); + const low12 = Number.parseInt(PROBE_SESSION_ID.slice(4, 16), 16) % 0x1000; + expect(low12).toBeGreaterThanOrEqual(0xfc0); + expect(low12).toBeLessThanOrEqual(0xffe); + }); + + it("an excerpt never carries the key, and is at most 200 characters", () => { + expect(excerpt('{"error":"bad key test-key test-key"}', "test-key")).toBe( + '{"error":"bad key "}' + ); + expect(excerpt("x".repeat(500), "k")).toHaveLength(200); + // An empty key redacts nothing, rather than interleaving the marker. + expect(excerpt("abc", "")).toBe("abc"); + }); +}); + +describe("keyed probes · through the offline gateway's own judge", () => { + /** What the mock gateway answers one keyed cell, read as the live suite reads it. */ + const judged = ( + cell: MatrixCell, + identity: Readonly> + ): { cell: string; verdict: string } => { + const keyed = keyedCell(cell, "sk-probe-0123456789abcdef", identity); + const url = new URL(keyed.url); + const { reply } = handleGatewayRequest({ + bodyText: JSON.stringify(keyed.body), + headers: keyed.headers, + method: keyed.method, + url: `${url.pathname}${url.search}`, + }); + return { + cell: cellKey(cell), + verdict: keyedVerdictOf([{ bodyText: reply.body, status: reply.status }]), + }; + }; + + const cells = keylessMatrix({ + go: OPENCODE_GO_CATALOG, + zen: OPENCODE_ZEN_CATALOG, + }); + const planeRows = ( + plane: MatrixPlane, + identity: Readonly> + ) => + cells + .filter((cell) => cell.plane === plane) + .map((cell) => judged(cell, identity)); + + it.for(["opencode", "opencode-go"] as const)( + "%s: with the plugin's identity every default cell is served and the plane passes; without it the session is refused and the plane fails", + (plane) => { + const rows = planeRows(plane, PROBE_IDENTITY); + for (const row of rows) { + if (TODAY_DISPATCH.has(row.cell)) { + expect(row.verdict, row.cell).toBe("served"); + } + } + expect(planeOutcome(rows)).toEqual({ kind: "pass" }); + + // The defect a session-less probe had: every routed cell refused, read + // as a failure that is not flip evidence — never as a pass. + const bare = planeRows(plane, {}); + expect( + bare + .filter((row) => TODAY_DISPATCH.has(row.cell)) + .map((row) => row.verdict) + ).toEqual(Array.from({ length: 4 }, () => "session-refused")); + const outcome = planeOutcome(bare); + expect(outcome.kind).toBe("fail"); + expect(outcome.kind === "fail" ? outcome.reason : "").toContain( + "not flip evidence" + ); + } + ); +}); + +describe("keyed probes · reading an answer", () => { + it.each([ + [200, "{}", "served"], + [204, "", "served"], + [404, "", "path-gone"], + [404, '{"error":{"message":"no route","type":"not_found"}}', "path-gone"], + [ + 404, + '{"type":"error","error":{"type":"not_found_error","message":"model: x"}}', + "model-gone", + ], + // Anthropic's generic 404 — an unknown route included — names no model. + [ + 404, + '{"type":"error","error":{"type":"not_found_error","message":"Not Found"}}', + "path-gone", + ], + [404, '{"error":{"code":"model_not_found"}}', "model-gone"], + [ + 404, + '{"error":{"message":"models/gemini-x is not found for API version v1beta"}}', + "model-gone", + ], + [410, "", "model-gone"], + [400, '{"error":{"message":"Model is unavailable"}}', "model-gone"], + [400, '{"error":{"type":"ModelProtocolUnsupported"}}', "wrong-protocol"], + // The vendor's withdrawal, by type at any status (the rule a turn is + // read by): never a gone path, never a shape that parsed the request. + [ + 400, + '{"type":"error","error":{"type":"ModelDeprecated","message":"Model x has been deprecated."}}', + "model-gone", + ], + [404, '{"error":{"type":"ModelDeprecated"}}', "model-gone"], + [ + 410, + '{"type":"error","error":{"type":"ModelDeprecated","message":"Model x has been deprecated."}}', + "model-gone", + ], + [400, '{"error":{"type":"ModelDeprecatedish"}}', "path-exists"], + [404, '{"error":{"type":"ModelDeprecatedish"}}', "path-gone"], + [200, '{"choices":[{"message":{"content":"ModelDeprecated"}}]}', "served"], + [400, '{"error":{"message":"max_output_tokens too small"}}', "path-exists"], + [ + 400, + '{"error":{"message":"MissingSessionID","type":"missing_session_id"}}', + "session-refused", + ], + [400, "missing x-opencode-session header", "session-refused"], + [403, '{"error":{"type":"FreeTierError"}}', "session-refused"], + [401, '{"error":{"type":"AuthError"}}', "auth-refused"], + [403, "", "auth-refused"], + [403, '{"error":{"message":"Model access is disabled"}}', "account"], + [401, "Insufficient account funds", "account"], + [402, "", "account"], + [429, "", "rate-limited"], + [500, "", "other"], + [502, "", "other"], + [500, '{"error":{"message":"internal error"}}', "other"], + // The upstream busy: the gateway may have routed it, or answered first. + [503, "", "upstream-busy"], + [ + 529, + '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}', + "upstream-busy", + ], + [ + 503, + '{"error":{"code":503,"message":"The model is overloaded. Please try again later.","status":"UNAVAILABLE"}}', + "upstream-busy", + ], + [ + 500, + '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}', + "upstream-busy", + ], + // Each busy condition alone decides its row: a bare 529, and a Gemini + // overload the gateway re-wraps as a 500 or a 502, by either of its two + // markers. + [529, "", "upstream-busy"], + [502, '{"error":{"status":"UNAVAILABLE"}}', "upstream-busy"], + [500, '{"error":{"message":"The model is overloaded."}}', "upstream-busy"], + // The marker is case-sensitive, and a plain 5xx is not busy: a proxy's + // "Service Unavailable" page and a gateway timeout stay `other`. + [502, "Service Unavailable", "other"], + [504, "", "other"], + [301, "", "redirected"], + [302, "", "redirected"], + [308, "", "redirected"], + ] as const)("%i %s → %s", (status, bodyText, verdict) => { + expect(classifyKeyedProbe({ bodyText, status })).toBe(verdict); + }); + + const answer = (status: number, bodyText = "") => ({ bodyText, status }); + + it.each([ + ["one 200", [answer(200)], "served"], + ["one 404", [answer(404)], "path-gone"], + [ + "429, then no answer (the measured rate-limit hang)", + [answer(429), undefined], + "rate-limited", + ], + ["429, then 429", [answer(429), answer(429)], "rate-limited"], + ["429, then 200", [answer(429), answer(200)], "served"], + ["no answer, twice", [undefined, undefined], "transport-error"], + ["500, then no answer", [answer(500), undefined], "transport-error"], + // Only a 429 makes no answer a rate limit: a busy upstream and then no + // answer measured nothing, as a 500 and then no answer did not. + ["529, then no answer", [answer(529), undefined], "transport-error"], + ["503, then no answer", [answer(503), undefined], "transport-error"], + ["500, then 500", [answer(500), answer(500)], "other"], + ["529, then 503", [answer(529), answer(503)], "upstream-busy"], + ["529, then 200", [answer(529), answer(200)], "served"], + ["no answer, then 401", [undefined, answer(401)], "auth-refused"], + ] as const)("attempts: %s → %s", (_name, attempts, verdict) => { + expect(keyedVerdictOf(attempts)).toBe(verdict); + }); + + it("which verdicts fail the job, and which of them are flip evidence", () => { + expect([...FLIP_EVIDENCE].toSorted()).toEqual([ + "auth-refused", + "path-gone", + "wrong-protocol", + ]); + for (const verdict of [ + "path-gone", + "wrong-protocol", + "auth-refused", + "other", + "redirected", + "session-refused", + "transport-error", + "no-model", + ]) { + expect(breaksDefault(verdict), verdict).toBe(true); + } + for (const verdict of [ + "served", + "path-exists", + "model-gone", + "account", + "rate-limited", + "upstream-busy", + ]) { + expect(breaksDefault(verdict), verdict).toBe(false); + } + }); +}); + +describe("keyed probes · what a plane decides", () => { + /** A plane's twelve rows (Zen unless named): each default cell's verdict, every legacy and root cell `rest`. */ + const plane = ( + defaults: readonly string[], + rest = "path-gone", + name: "opencode" | "opencode-go" = "opencode" + ): { cell: string; verdict: string }[] => { + const modern = [...TODAY_DISPATCH].filter((cell) => + cell.startsWith(`${name} `) + ); + return [ + ...modern.map((cell, index) => ({ + cell, + verdict: defaults[index] ?? "served", + })), + ...modern.flatMap((cell) => + ["legacy", "root"].map((family) => ({ + cell: cell.replace(/ modern$/u, ` ${family}`), + verdict: rest, + })) + ), + ]; + }; + + it.each([ + [ + "default cells all account, a root path-gone", + plane(["account", "account", "account", "account"]), + "skip", + ], + [ + "default cells all model-gone", + plane(["model-gone", "model-gone", "model-gone", "model-gone"]), + "skip", + ], + [ + "default cells all path-exists (a parsed but refused body)", + plane(["path-exists", "path-exists", "path-exists", "path-exists"]), + "skip", + ], + [ + "default cells all rate-limited", + plane(["rate-limited", "rate-limited", "rate-limited", "rate-limited"]), + "skip", + ], + [ + "default cells all upstream-busy", + plane([ + "upstream-busy", + "upstream-busy", + "upstream-busy", + "upstream-busy", + ]), + "skip", + ], + [ + "one default upstream-busy, the rest served", + plane(["served", "upstream-busy", "served", "served"]), + "pass", + ], + [ + "one default path-gone, the rest account", + plane(["path-gone", "account", "account", "account"]), + "fail", + ], + [ + "one default transport-error, the rest served", + plane(["served", "served", "served", "transport-error"]), + "fail", + ], + [ + "one default served, the rest account; every other shape broken", + plane(["served", "account", "account", "account"], "other"), + "pass", + ], + [ + "every default served", + plane(["served", "served", "served", "served"]), + "pass", + ], + [ + "no default cell at all", + [{ cell: "opencode openai-completions legacy", verdict: "served" }], + "fail", + ], + ] as const)("%s → %s", (_name, rows, kind) => { + expect(planeOutcome(rows).kind).toBe(kind); + }); + + /** `rows` with each named cell's verdict replaced. */ + const withCells = ( + rows: readonly { cell: string; verdict: string }[], + verdicts: Readonly> + ): { cell: string; verdict: string }[] => + rows.map((row) => ({ ...row, verdict: verdicts[row.cell] ?? row.verdict })); + const ZEN_CHAT = "opencode openai-completions"; + + it.each([ + // A default cell's model-gone is read beside the same model on the + // other shapes: the gateway answers "Model is unavailable" for an id + // sent where it is not served too, so a model another shape accepts is + // the default family refusing it. + [ + "modern model-gone, the same model served on legacy", + { [`${ZEN_CHAT} legacy`]: "served" }, + "fail", + "(flip evidence", + ], + [ + // Nothing dispatches to the root, so no flip of the default reaches + // the shape that took the model: the plane fails, but not as evidence. + "modern model-gone, the same model a parsed body on root only", + { [`${ZEN_CHAT} root`]: "path-exists" }, + "fail", + "(not flip evidence: no flip reaches that shape)", + ], + [ + // The account's gate answers before the model is routed, so it says + // nothing about whether legacy serves the model. + "modern model-gone, the account's answer on legacy", + { [`${ZEN_CHAT} legacy`]: "account" }, + "pass", + undefined, + ], + [ + "modern model-gone, the same model gone on every shape", + { + [`${ZEN_CHAT} legacy`]: "model-gone", + [`${ZEN_CHAT} root`]: "model-gone", + }, + "pass", + undefined, + ], + ["modern model-gone, the other shapes' paths gone", {}, "pass", undefined], + ] as const)( + "a retired model, read across the shapes: %s → %s", + (_name, siblings, kind, reason) => { + const outcome = planeOutcome( + withCells(plane(["model-gone", "served", "served", "served"]), siblings) + ); + expect(outcome.kind).toBe(kind); + if (outcome.kind === "fail") { + expect(outcome.reason).toContain(reason); + if (reason?.startsWith("(not") === true) { + expect(outcome.reason).not.toContain("(flip evidence"); + } + expect(outcome.broken).toEqual([ + { cell: `${ZEN_CHAT} modern`, verdict: "model-gone" }, + ]); + } + } + ); + + it.each([ + [ + "auth-refused, the key refused on every shape (a revoked CI secret)", + "auth-refused", + "auth-refused", + "not flip evidence: check the CI secret", + ], + [ + // A gone path never measured the key: the gateway answers 404 before + // it authenticates. + "auth-refused, the other shapes' paths gone", + "auth-refused", + "path-gone", + "not flip evidence: nothing else measured the key", + ], + [ + "auth-refused, the other shapes rate-limited", + "auth-refused", + "rate-limited", + "not flip evidence: nothing else measured the key", + ], + [ + // A busy answer may come from the gateway's front, before the key is + // read, so it is not the key accepted. + "auth-refused, the other shapes' upstream busy", + "auth-refused", + "upstream-busy", + "not flip evidence: nothing else measured the key", + ], + [ + "auth-refused, the same key served on legacy", + "auth-refused", + "served", + "(flip evidence", + ], + [ + "path-gone on every shape", + "path-gone", + "path-gone", + "not flip evidence: a flip would not serve it", + ], + [ + "path-gone, the path existing on legacy", + "path-gone", + "path-exists", + "(flip evidence", + ], + [ + // The model's protocol refused everywhere is the model, not the family. + "wrong-protocol on every shape", + "wrong-protocol", + "wrong-protocol", + "not flip evidence: a flip would not serve it", + ], + [ + "wrong-protocol, the same request served on legacy", + "wrong-protocol", + "served", + "(flip evidence", + ], + ] as const)( + "a refused protocol, key or path is flip evidence only beside the shape a flip reaches, having accepted it: %s", + (_name, verdict, rest, reason) => { + // Every cell of the plane answers `rest` but the first default cell. + const outcome = planeOutcome(plane([verdict, rest, rest, rest], rest)); + expect(outcome.kind).toBe("fail"); + expect(outcome.kind === "fail" ? outcome.reason : "").toContain(reason); + if (reason.startsWith("not")) { + expect(outcome.kind === "fail" ? outcome.reason : "").not.toContain( + "(flip evidence" + ); + } + } + ); + + /** The fail reason of `rows`' outcome, or `""` when it did not fail. */ + const failReason = (rows: readonly { cell: string; verdict: string }[]) => { + const outcome = planeOutcome(rows); + expect(outcome.kind).toBe("fail"); + return outcome.kind === "fail" ? outcome.reason : ""; + }; + const GO_GOOGLE = "opencode-go google-generative-ai"; + const GO_CHAT = "opencode-go openai-completions"; + + it.each([ + [ + // A flip of the default sends Zen's completions to legacy, which is + // gone too: only the root, which nothing reaches, accepted it. + "Zen completions: modern and legacy path-gone, root served", + withCells(plane(["path-gone", "served", "served", "served"]), { + [`${ZEN_CHAT} root`]: "served", + }), + false, + ], + [ + "Zen completions: modern path-gone, legacy served", + withCells(plane(["path-gone", "served", "served", "served"]), { + [`${ZEN_CHAT} legacy`]: "served", + }), + true, + ], + [ + // Go's Google stays on its modern cell under every family, so its + // `/zen/go` sibling is a shape no setting sends to. + "Go Google: modern path-gone, its /zen/go sibling path-exists", + withCells( + plane( + ["served", "served", "served", "path-gone"], + "path-gone", + "opencode-go" + ), + { [`${GO_GOOGLE} legacy`]: "path-exists" } + ), + false, + ], + [ + "Go Google: modern model-gone, the same model a parsed body on root", + withCells( + plane( + ["served", "served", "served", "model-gone"], + "path-gone", + "opencode-go" + ), + { [`${GO_GOOGLE} root`]: "path-exists" } + ), + false, + ], + [ + "Go Google: modern auth-refused, its /zen/go sibling served", + withCells( + plane( + ["served", "served", "served", "auth-refused"], + "path-gone", + "opencode-go" + ), + { [`${GO_GOOGLE} legacy`]: "served" } + ), + false, + ], + [ + "Go completions: modern path-gone, legacy served", + withCells( + plane( + ["path-gone", "served", "served", "served"], + "path-gone", + "opencode-go" + ), + { [`${GO_CHAT} legacy`]: "served" } + ), + true, + ], + [ + "Go completions: modern wrong-protocol, root served only", + withCells( + plane( + ["wrong-protocol", "served", "served", "served"], + "path-gone", + "opencode-go" + ), + { [`${GO_CHAT} root`]: "served" } + ), + false, + ], + ] as const)( + "flip evidence needs the cell `endpoint: legacy` dispatches to, a different one, to have accepted: %s", + (_name, rows, evidence) => { + const reason = failReason(rows); + if (evidence) { + expect(reason).toContain("(flip evidence"); + } else { + expect(reason).not.toContain("(flip evidence"); + expect(reason).toContain( + "(not flip evidence: no flip reaches that shape)" + ); + } + } + ); + + const CI_SECRET = "(not flip evidence: check the CI secret)"; + const KEY_UNMEASURED = "(not flip evidence: nothing else measured the key)"; + const FLIP_UNMEASURED = + "(not flip evidence this run: the shape a flip reaches measured nothing; re-run)"; + const NO_FLIP_SERVES = "(not flip evidence: a flip would not serve it)"; + + it.each([ + // A sibling that never read the key — a gone path, a rate limit, a busy + // upstream — neither confirms nor vetoes the key's refusal: the reason is + // read from the siblings that did read it. + [ + "modern auth-refused, legacy refused the key, root path-gone", + "auth-refused", + { legacy: "auth-refused", root: "path-gone" }, + CI_SECRET, + ], + [ + "modern auth-refused, legacy refused the key, root rate-limited", + "auth-refused", + { legacy: "auth-refused", root: "rate-limited" }, + CI_SECRET, + ], + [ + "modern auth-refused, legacy refused the key, root upstream-busy", + "auth-refused", + { legacy: "auth-refused", root: "upstream-busy" }, + CI_SECRET, + ], + [ + "modern auth-refused, legacy path-gone, root refused the key", + "auth-refused", + { legacy: "path-gone", root: "auth-refused" }, + CI_SECRET, + ], + [ + "modern auth-refused, legacy rate-limited, root refused the key", + "auth-refused", + { legacy: "rate-limited", root: "auth-refused" }, + CI_SECRET, + ], + [ + "modern auth-refused, legacy upstream-busy, root path-gone", + "auth-refused", + { legacy: "upstream-busy", root: "path-gone" }, + KEY_UNMEASURED, + ], + // A protocol or a path is decided by the shape a flip reaches: when that + // one measured nothing, the run names no cause. + [ + "modern wrong-protocol, legacy rate-limited, root path-gone", + "wrong-protocol", + { legacy: "rate-limited", root: "path-gone" }, + FLIP_UNMEASURED, + ], + [ + "modern wrong-protocol, legacy upstream-busy, root path-gone", + "wrong-protocol", + { legacy: "upstream-busy", root: "path-gone" }, + FLIP_UNMEASURED, + ], + [ + "modern wrong-protocol, legacy transport-error, root wrong-protocol", + "wrong-protocol", + { legacy: "transport-error", root: "wrong-protocol" }, + FLIP_UNMEASURED, + ], + [ + "modern path-gone, legacy rate-limited, root path-gone", + "path-gone", + { legacy: "rate-limited", root: "path-gone" }, + FLIP_UNMEASURED, + ], + [ + "modern path-gone, legacy upstream-busy, root path-gone", + "path-gone", + { legacy: "upstream-busy", root: "path-gone" }, + FLIP_UNMEASURED, + ], + [ + "modern path-gone, legacy transport-error, root path-gone", + "path-gone", + { legacy: "transport-error", root: "path-gone" }, + FLIP_UNMEASURED, + ], + [ + "modern wrong-protocol, legacy path-gone, root rate-limited", + "wrong-protocol", + { legacy: "path-gone", root: "rate-limited" }, + NO_FLIP_SERVES, + ], + [ + "modern path-gone, legacy refused the key, root upstream-busy", + "path-gone", + { legacy: "auth-refused", root: "upstream-busy" }, + NO_FLIP_SERVES, + ], + // The account's gate answers before the format check and the model's + // routing (protocol-routing.e2e.ts), so beside a refused protocol it + // measured nothing; beside a refused key or a gone path it shows the key + // was read and the path exists. + [ + "modern wrong-protocol, legacy the account's", + "wrong-protocol", + { legacy: "account", root: "path-gone" }, + FLIP_UNMEASURED, + ], + [ + "modern path-gone, legacy the account's", + "path-gone", + { legacy: "account", root: "path-gone" }, + "(flip evidence", + ], + [ + "modern auth-refused, legacy the account's", + "auth-refused", + { legacy: "account", root: "path-gone" }, + "(flip evidence", + ], + [ + "modern wrong-protocol, root the account's only", + "wrong-protocol", + { legacy: "path-gone", root: "account" }, + NO_FLIP_SERVES, + ], + ] as const)( + "a refusal is read from the siblings that measured it: %s", + (_name, verdict, siblings, reason) => { + const rows = withCells(plane([verdict, "served", "served", "served"]), { + [`${ZEN_CHAT} legacy`]: siblings.legacy, + [`${ZEN_CHAT} root`]: siblings.root, + }); + const failed = failReason(rows); + expect(failed).toContain(reason); + for (const other of [ + CI_SECRET, + KEY_UNMEASURED, + FLIP_UNMEASURED, + NO_FLIP_SERVES, + "(flip evidence", + ]) { + if (other !== reason) { + expect(failed).not.toContain(other); + } + } + } + ); + + it("Go's Google, which no flip moves, is never a re-run: a flip would not serve it whatever its siblings answered", () => { + const failed = failReason( + plane( + ["served", "served", "served", "path-gone"], + "rate-limited", + "opencode-go" + ) + ); + expect(failed).toContain(NO_FLIP_SERVES); + expect(failed).not.toContain(FLIP_UNMEASURED); + }); + + it("a flip target the run never probed measured nothing", () => { + const rows = plane(["wrong-protocol", "served", "served", "served"]).filter( + (row) => row.cell !== `${ZEN_CHAT} legacy` + ); + expect(failReason(rows)).toContain(FLIP_UNMEASURED); + }); + + it.each([ + ["legacy", "account"], + ["root", "account"], + ] as const)( + "a retired model beside the account's answer on %s fails nothing: the account's gate answers before the model is routed", + (family, verdict) => { + expect( + planeOutcome( + withCells(plane(["model-gone", "served", "served", "served"]), { + [`${ZEN_CHAT} ${family}`]: verdict, + }) + ).kind + ).toBe("pass"); + } + ); + + it("a busy upstream and then no answer fails a default cell as unreadable, not as flip evidence", () => { + const verdict = keyedVerdictOf([{ bodyText: "", status: 529 }, undefined]); + expect(verdict).toBe("transport-error"); + const outcome = planeOutcome( + plane(["served", verdict, "served", "served"]) + ); + expect(outcome.kind).toBe("fail"); + expect(outcome.kind === "fail" ? outcome.reason : "").toContain( + "gave no readable answer (not flip evidence)" + ); + }); + + it("a failure names its rows and says whether it is flip evidence", () => { + const evidence = planeOutcome( + plane(["wrong-protocol", "served"], "served") + ); + expect(evidence.kind).toBe("fail"); + if (evidence.kind === "fail") { + expect(evidence.broken.map((row) => row.verdict)).toEqual([ + "wrong-protocol", + ]); + expect(evidence.reason).toContain("(flip evidence"); + } + const unreadable = planeOutcome(plane(["other", "served"])); + expect(unreadable.kind === "fail" ? unreadable.reason : "").toContain( + "not flip evidence" + ); + const skipped = planeOutcome( + plane(["account", "rate-limited", "account", "account"]) + ); + expect(skipped.kind === "skip" ? skipped.reason : "").toContain( + "account, rate-limited, account, account" + ); + }); +}); + +/** A GitHub Actions expression, `${{ inner }}`, as the workflow spells it. */ +const expression = (inner: string): string => + ["$", "{{ ", inner, " }}"].join(""); + +describe("keyed probes · the CI wiring", () => { + const ci = readFileSync( + new URL("../.github/workflows/ci.yml", import.meta.url), + "utf8" + ); + const start = ci.indexOf("\n e2e:\n"); + const end = ci.indexOf("\n e2e-host:"); + const job = ci.slice(start, end); + /** The job's steps, split at each `- ` list item of its `steps:`. */ + const steps = job.split(/\n {6}- /u); + const tierStep = steps.find((step) => step.includes("id: tier")) ?? ""; + const testStep = + steps.find((step) => step.includes("run: pnpm run test:e2e")) ?? ""; + /** The scope pattern the tier step greps the changed paths with. */ + const scope = /grep -Eq '([^']+)'/u.exec(tierStep)?.[1]; + + it("the job and its tier step are where this test looks", () => { + expect(start).toBeGreaterThan(-1); + expect(end).toBeGreaterThan(start); + expect(tierStep).toContain("id: tier"); + expect(job).toContain("fetch-depth: 0"); + expect(scope).toBeDefined(); + }); + + it("the scope arms the probes on the host seam, the legacy URLs and protocols they send, the family the plugin dispatches to, the identity and anthropic-version they carry, and their own inputs, and on nothing else", () => { + // The ERE is JavaScript-compatible, so the pattern CI runs is the one + // tested here. + const armed = new RegExp(scope ?? "(?!)", "u"); + for (const path of [ + "src/host/route-claim.ts", + "src/host/nested/deep.ts", + "src/utils/protocols.ts", + "src/utils/plane-model.ts", + "src/identity.ts", + "src/utils/request-shape.ts", + "test/e2e/keyed-probes.e2e.ts", + "test/support/keyed-probes.ts", + "test/support/keyless-matrix.ts", + ".github/workflows/ci.yml", + ]) { + expect(armed.test(path), path).toBe(true); + } + for (const path of [ + "README.md", + "docs/redirect-planes.md", + "src/fetch-patch.ts", + "src/hostile.ts", + "src/utils/plane-model.test.ts", + // Named as left out in the tier step: covered offline and nightly. + "src/catalog-data.ts", + "src/models-catalog.ts", + "src/session.ts", + "src/utils/request-shape.test.ts", + "test/keyed-probes.test.ts", + "test/support/keyed-probes.ts.bak", + "x/src/host/a.ts", + ]) { + expect(armed.test(path), path).toBe(false); + } + }); + + it("a match arms `probes` and a miss `none`; schedule and dispatch arm `full`", () => { + expect(tierStep).toContain("schedule|workflow_dispatch) tier=full"); + // The arms in order: the grep, then `probes`, then `else`, then `none`. + // grep reads a here-string: piped, `grep -q` exiting on an early match + // can kill the writer with SIGPIPE, which pipefail reads as a miss. + expect(tierStep).toMatch( + /if grep -Eq '[^']+' <<<"\$changed"; then\n\s+tier=probes\n\s+else\n\s+tier=none\n\s+fi/u + ); + expect(tierStep).not.toMatch(/\|\s*grep/u); + }); + + it("every event the job runs on but schedule, dispatch and a pull request arms `none`: a push to main spends no key", () => { + const arms = [...tierStep.matchAll(/^\s+([^\s()]+)\)/gmu)].map( + (match) => match[1] + ); + expect(arms).toEqual(["schedule|workflow_dispatch", "pull_request", "*"]); + expect(tierStep).toMatch(/\n\s+\*\) tier=none ;;\n\s+esac/u); + }); + + it("the record is published and uploaded on every outcome, from the directory the suites write to, under the names they write", () => { + // The red run is the one that carries the flip evidence (§13), so the + // two steps that keep the record run `if: always()`. + const publish = + steps.find((step) => step.includes("GITHUB_STEP_SUMMARY")) ?? ""; + const upload = + steps.find((step) => step.includes("actions/upload-artifact")) ?? ""; + for (const step of [publish, upload]) { + expect(step).toMatch(/^\s*if: always\(\)$/mu); + } + const dir = /OPENCODE_E2E_RECORD_DIR: (.+)$/mu.exec(testStep)?.[1]; + expect(dir).toBe(`${expression("runner.temp")}/endpoint-families`); + expect(publish).toContain(`RECORD_DIR: ${dir ?? ""}`); + expect(upload).toContain(`path: ${dir ?? ""}`); + expect(upload).toContain("name: endpoint-families"); + // The publisher reads the two names the suites write. + expect(publish).toMatch(/for file in keyless\.md keyed\.md; do/u); + expect(liveSource("keyless-matrix.e2e.ts")).toContain( + 'join(dir, "keyless.md")' + ); + expect(liveSource("keyed-probes.e2e.ts")).toContain( + 'join(dir, "keyed.md")' + ); + }); + + it("each suite rewrites its record right after it records a cell, so a job killed at its time limit keeps what it measured", () => { + /** What follows each `recorded.push(…);` call, from its balanced close. */ + const afterEachPush = (source: string): string[] => + [...source.matchAll(/recorded\.push\(/gu)].map((match) => { + let depth = 0; + let index = match.index + match[0].length - 1; + do { + const char = source[index]; + if (char === "(") { + depth += 1; + } else if (char === ")") { + depth -= 1; + } + index += 1; + } while (depth > 0 && index < source.length); + return source.slice(index, index + 80); + }); + for (const [file, pushes] of [ + ["keyless-matrix.e2e.ts", 1], + ["keyed-probes.e2e.ts", 2], + ] as const) { + const source = liveSource(file); + const after = afterEachPush(source); + expect(after, file).toHaveLength(pushes); + for (const rest of after) { + expect(rest, file).toMatch(/^;\s*await persist\(false\);/u); + } + } + // The free probe's line is rewritten as soon as it is decided. + expect(liveSource("keyless-matrix.e2e.ts")).toMatch( + /\n\s*free = [^;]+;\s*await persist\(false\);/u + ); + }); + + it("the test step names the tier and passes the keys only when it is not `none`", () => { + // Unset means `full`, so a step that dropped the tier line would arm the + // full keyed suites on every pull request that arms the probes. + for (const line of [ + `OPENCODE_E2E_KEYED: ${expression("steps.tier.outputs.keyed")}`, + `OPENCODE_API_KEY: ${expression("steps.tier.outputs.keyed != 'none' && secrets.OPENCODE_API_KEY || ''")}`, + `OPENCODE_GO_API_KEY: ${expression("steps.tier.outputs.keyed != 'none' && secrets.OPENCODE_GO_API_KEY || ''")}`, + "OPENCODE_E2E_RECORD_DIR", + ]) { + expect(testStep, line).toContain(line); + } + // And no other step of the job sees a key. + for (const step of steps) { + if (step !== testStep) { + expect(step).not.toContain("secrets.OPENCODE"); + } + } + }); +}); + +describe("keyed probes · the variants on the default completions cell (L4, L5, L9, L10)", () => { + const models = { reasoning: "zen-think", representative: "zen-chat" }; + + it("each variant is the default completions cell, keyed, changed by its own header or body field and nothing else", () => { + const cells = keyedVariantCells("opencode", models, "sk-probe"); + expect(cells.map((cell) => cell.variant.id)).toEqual([ + "baseline", + "request-id", + "project-folder", + "project-digest", + "reasoning-default", + "reasoning-none", + ]); + const [baseline, request, folder, digest, reasoning, none] = cells; + // The baseline is exactly the matrix's keyed modern completions cell. + const matrixCell = keylessMatrix({ + go: GO, + zen: [spec("zen-chat"), ...ZEN.slice(1)], + }).find( + (cell) => + cell.plane === "opencode" && + cell.protocol === "openai-completions" && + cell.family === "modern" + ); + if (matrixCell === undefined || baseline === undefined) { + throw new Error("no modern completions cell"); + } + expect(baseline).toEqual({ + ...keyedCell(matrixCell, "sk-probe"), + variant: baseline.variant, + }); + expect(baseline.url).toBe( + "https://opencode.ai/inference/openai/v1/chat/completions" + ); + const without = ( + headers: Readonly> | undefined, + name: string + ) => + Object.fromEntries( + Object.entries(headers ?? {}).filter(([header]) => header !== name) + ); + expect(baseline?.headers["x-opencode-request"]).toBeUndefined(); + expect(request?.headers["x-opencode-request"]).toBe(PROBE_REQUEST_ID); + expect(without(request?.headers, "x-opencode-request")).toEqual( + baseline?.headers + ); + expect(baseline?.headers["x-opencode-project"]).toBe("global"); + expect(folder?.headers["x-opencode-project"]).toBe(PROBE_PROJECTS.folder); + expect(digest?.headers["x-opencode-project"]).toMatch(/^[0-9a-f]{40}$/u); + for (const cell of [request, folder, digest]) { + expect(cell?.body).toEqual(baseline?.body); + expect(cell?.model).toBe("zen-chat"); + } + expect(without(folder?.headers, "x-opencode-project")).toEqual( + without(baseline?.headers, "x-opencode-project") + ); + // The L10 pair: one model, and the one field between them. + expect(reasoning?.model).toBe("zen-think"); + expect(reasoning?.headers).toEqual(baseline?.headers); + expect(reasoning?.body).toEqual( + keyedBody("openai-completions", "zen-think") + ); + expect(none?.body).toEqual({ + ...reasoning?.body, + reasoning_effort: "none", + }); + expect(none?.headers).toEqual(reasoning?.headers); + }); + + it("the request id is the client's own msg_ shape, and the folder names no real path", () => { + expect(isOpenCodeMessageId(PROBE_REQUEST_ID)).toBe(true); + expect(isOpenCodeMessageId(newOpenCodeMessageId())).toBe(true); + expect(PROBE_PROJECTS.folder).not.toMatch(/[/\\]/u); + }); + + it("a variant with no model keeps no body, and is never sent", () => { + const cells = keyedVariantCells( + "opencode-go", + { reasoning: undefined, representative: "go-chat" }, + "sk-probe" + ); + expect( + cells + .filter((cell) => cell.body === undefined) + .map((cell) => cell.variant.id) + ).toEqual(["reasoning-default", "reasoning-none"]); + expect(cells[0]?.url).toBe( + "https://opencode.ai/inference/go/openai/v1/chat/completions" + ); + }); + + it("the reasoning model is the first paid, current completions model that reasons", () => { + const thinking = (id: string, provider_npm?: string): CatalogModelSpec => ({ + ...spec(id, provider_npm), + reasoning: true, + }); + expect( + reasoningCompletionsModel( + [ + spec("plain"), + thinking("resp", "@ai-sdk/openai"), + thinking("think-free"), + { ...thinking("think-paid-but-free"), is_free: true }, + thinking("retired"), + thinking("think"), + thinking("think-2"), + ], + (id) => id === "retired" + ) + ).toBe("think"); + expect(reasoningCompletionsModel([spec("plain")])).toBeUndefined(); + // The bundled catalogs carry one on each plane. + expect(reasoningCompletionsModel(OPENCODE_ZEN_CATALOG)).toBeDefined(); + expect(reasoningCompletionsModel(OPENCODE_GO_CATALOG)).toBeDefined(); + }); + + it("every variant names its lesson and its label once", () => { + expect(new Set(KEYED_VARIANTS.map((variant) => variant.id)).size).toBe( + KEYED_VARIANTS.length + ); + expect(KEYED_VARIANTS.map((variant) => variant.lesson)).toEqual([ + "L4, L5", + "L4", + "L5", + "L5", + "L10", + "L10", + ]); + }); + + it("the offline gateway serves the baseline with the plugin's identity", () => { + const representative = keylessMatrix({ + go: OPENCODE_GO_CATALOG, + zen: OPENCODE_ZEN_CATALOG, + }).find( + (cell) => + cell.plane === "opencode" && + cell.protocol === "openai-completions" && + cell.family === "modern" + )?.model; + const [baseline] = keyedVariantCells( + "opencode", + { reasoning: undefined, representative }, + "sk-probe-0123456789abcdef" + ); + if (baseline?.body === undefined) { + throw new Error("the bundled catalog has no completions model"); + } + const url = new URL(baseline.url); + const { reply } = handleGatewayRequest({ + bodyText: JSON.stringify(baseline.body), + headers: baseline.headers, + method: baseline.method, + url: `${url.pathname}${url.search}`, + }); + expect( + keyedVerdictOf([{ bodyText: reply.body, status: reply.status }]) + ).toBe("served"); + }); + + it("reads every 429's Retry-After, and names one that carried none (L9)", () => { + expect( + retryAfterOf([ + { bodyText: "", retryAfter: "30", status: 429 }, + { bodyText: "", status: 429 }, + ]) + ).toEqual(["30", null]); + expect( + retryAfterOf([ + undefined, + { bodyText: "", retryAfter: "30", status: 503 }, + { bodyText: "", status: 200 }, + ]) + ).toEqual([]); + }); +}); diff --git a/test/keyless-matrix.test.ts b/test/keyless-matrix.test.ts index dd32d3c..7862e3e 100644 --- a/test/keyless-matrix.test.ts +++ b/test/keyless-matrix.test.ts @@ -3,7 +3,9 @@ * * The live half (`test/e2e/keyless-matrix.e2e.ts`) cannot run in a sandbox * without the vendor's hosts, so everything it decides is decided here: which - * sixteen URLs are probed, with which bodies, and how an answer is read. The + * twenty-four URLs are probed (three URL shapes), with which bodies, which + * cells each `endpoint` setting dispatches to, which free model the keyless + * free probe sends, and how each answer is read. The * URLs are written out from the vendor's endpoint layout, not derived from the * module, and the bundled-catalog case checks invariants rather than model * names, so the nightly catalog refresh cannot turn it red. @@ -15,13 +17,31 @@ import { describe, expect, it } from "vitest"; import { HOST_CELLS, PLANES, S0_TODAY_BY_CELL } from "./support/host.ts"; import { cellKey, + classifyFreeProbe, classifyKeylessProbe, + freeProbeCells, + freeProbeMovesOn, + freeProbeOutcome, + freeProbeVerdictOf, + goneDefaultRows, isFreeSpec, + isFreeUsageLimit, isModelGoneMessage, isTransientProbeAnswer, + isUpstreamBusy, keylessMatrix, + LEGACY_DISPATCH, + type ProbeAnswer, + runFreeProbe, + MATRIX_FAMILIES, MATRIX_PROTOCOLS, type MatrixCell, + type MatrixFamily, + redirectPathOf, + SYSTEM_ONE_URLS, + systemOneCells, + unreadRows, + verdictOfAttempts, matrixUrl, TODAY_DISPATCH, } from "./support/keyless-matrix.ts"; @@ -84,14 +104,21 @@ const CREDENTIAL_HEADERS = new Set([ ]); describe("keyless matrix · the cells", () => { - it("is sixteen cells with exactly these URLs and models", () => { + it("is twenty-four cells over three URL shapes with exactly these URLs and models", () => { + expect(MATRIX_FAMILIES).toEqual(["legacy", "root", "modern"]); const cells = keylessMatrix({ go: GO, isDeprecated: (plane, id) => plane === "opencode" && id === "zen-old-resp", zen: ZEN, }); - expect(cells).toHaveLength(16); + expect(cells).toHaveLength(24); + // Plane, then protocol, then the three shapes in order. + expect(cells.slice(0, 3).map(key)).toEqual([ + "opencode openai-completions legacy", + "opencode openai-completions root", + "opencode openai-completions modern", + ]); expect( Object.fromEntries( cells.map((cell) => [key(cell), [cell.url, cell.model]]) @@ -101,6 +128,10 @@ describe("keyless matrix · the cells", () => { "https://opencode.ai/zen/v1/chat/completions", "zen-chat", ], + "opencode openai-completions root": [ + "https://opencode.ai/inference/v1/chat/completions", + "zen-chat", + ], "opencode openai-completions modern": [ "https://opencode.ai/inference/openai/v1/chat/completions", "zen-chat", @@ -109,6 +140,10 @@ describe("keyless matrix · the cells", () => { "https://opencode.ai/zen/v1/responses", "zen-resp", ], + "opencode openai-responses root": [ + "https://opencode.ai/inference/v1/responses", + "zen-resp", + ], "opencode openai-responses modern": [ "https://opencode.ai/inference/openai/v1/responses", "zen-resp", @@ -117,6 +152,10 @@ describe("keyless matrix · the cells", () => { "https://opencode.ai/zen/v1/messages", "zen-claude", ], + "opencode anthropic-messages root": [ + "https://opencode.ai/inference/v1/messages", + "zen-claude", + ], "opencode anthropic-messages modern": [ "https://opencode.ai/inference/anthropic/v1/messages", "zen-claude", @@ -125,6 +164,10 @@ describe("keyless matrix · the cells", () => { "https://opencode.ai/zen/v1/models/zen-gemini:streamGenerateContent?alt=sse", "zen-gemini", ], + "opencode google-generative-ai root": [ + "https://opencode.ai/inference/v1/models/zen-gemini:streamGenerateContent?alt=sse", + "zen-gemini", + ], "opencode google-generative-ai modern": [ "https://opencode.ai/inference/google/v1beta/models/zen-gemini:streamGenerateContent?alt=sse", "zen-gemini", @@ -133,6 +176,10 @@ describe("keyless matrix · the cells", () => { "https://opencode.ai/zen/go/v1/chat/completions", "go-chat", ], + "opencode-go openai-completions root": [ + "https://opencode.ai/inference/go/v1/chat/completions", + "go-chat", + ], "opencode-go openai-completions modern": [ "https://opencode.ai/inference/go/openai/v1/chat/completions", "go-chat", @@ -141,6 +188,10 @@ describe("keyless matrix · the cells", () => { "https://opencode.ai/zen/go/v1/responses", "go-resp", ], + "opencode-go openai-responses root": [ + "https://opencode.ai/inference/go/v1/responses", + "go-resp", + ], "opencode-go openai-responses modern": [ "https://opencode.ai/inference/go/openai/v1/responses", "go-resp", @@ -149,6 +200,10 @@ describe("keyless matrix · the cells", () => { "https://opencode.ai/zen/go/v1/messages", "go-claude", ], + "opencode-go anthropic-messages root": [ + "https://opencode.ai/inference/go/v1/messages", + "go-claude", + ], "opencode-go anthropic-messages modern": [ "https://opencode.ai/inference/go/anthropic/v1/messages", "go-claude", @@ -157,6 +212,10 @@ describe("keyless matrix · the cells", () => { "https://opencode.ai/zen/go/v1/models/go-gemini:streamGenerateContent", "go-gemini", ], + "opencode-go google-generative-ai root": [ + "https://opencode.ai/inference/go/v1/models/go-gemini:streamGenerateContent?alt=sse", + "go-gemini", + ], "opencode-go google-generative-ai modern": [ "https://opencode.ai/inference/go/google/v1beta/models/go-gemini:streamGenerateContent?alt=sse", "go-gemini", @@ -216,7 +275,7 @@ describe("keyless matrix · the cells", () => { zen: [spec("zen-beam", undefined, true)], }); const zenCells = onlyFree.filter((cell) => cell.plane === "opencode"); - expect(zenCells).toHaveLength(8); + expect(zenCells).toHaveLength(12); expect(zenCells.filter((cell) => cell.model !== undefined)).toEqual([]); expect(isFreeSpec(spec("paid"))).toBe(false); expect(isFreeSpec(spec("x", undefined, true))).toBe(true); @@ -231,6 +290,7 @@ describe("keyless matrix · the cells", () => { const missing = cells.filter((cell) => cell.model === undefined); expect(missing.map(key)).toEqual([ "opencode-go google-generative-ai legacy", + "opencode-go google-generative-ai root", "opencode-go google-generative-ai modern", ]); for (const cell of missing) { @@ -238,12 +298,12 @@ describe("keyless matrix · the cells", () => { } }); - it("on the bundled catalogs: sixteen cells, each the first PAID model of its protocol, and none can carry a key", () => { + it("on the bundled catalogs: twenty-four cells, each the first PAID model of its protocol, and none can carry a key", () => { const cells = keylessMatrix({ go: OPENCODE_GO_CATALOG, zen: OPENCODE_ZEN_CATALOG, }); - expect(cells).toHaveLength(16); + expect(cells).toHaveLength(24); for (const cell of cells) { const catalog = cell.plane === "opencode" ? OPENCODE_ZEN_CATALOG : OPENCODE_GO_CATALOG; @@ -280,8 +340,8 @@ describe("keyless matrix · the cells", () => { describe("keyless matrix · the cells the live run asserts", () => { /** * The matrix cells one host cell's characterized turns were sent to, and - * the URLs that match no cell (the root family, `/inference(/go)/v1`, which - * the matrix does not probe yet: P-R2 adds it). + * the URLs that match no cell of any of the three shapes, root included — + * a turn sent anywhere the matrix does not probe. */ const reachedOn = (cell: (typeof HOST_CELLS)[number]) => { const reached = new Set(); @@ -293,13 +353,11 @@ describe("keyless matrix · the cells the live run asserts", () => { expect(PLANES, rowKey).toContain(plane); for (const sent of row.sent) { const matches = MATRIX_PROTOCOLS.flatMap((protocol) => - (["legacy", "modern"] as const) - .filter( - (family) => - new URL(matrixUrl(plane, protocol, family, model)).pathname === - new URL(sent).pathname - ) - .map((family) => cellKey({ family, plane, protocol })) + MATRIX_FAMILIES.filter( + (family) => + new URL(matrixUrl(plane, protocol, family, model)).pathname === + new URL(sent).pathname + ).map((family) => cellKey({ family, plane, protocol })) ); expect(matches.length, `${cell} ${rowKey} ${sent}`).toBeLessThan(2); if (matches[0] === undefined) { @@ -329,14 +387,16 @@ describe("keyless matrix · the cells the live run asserts", () => { expect(reached, cell).toEqual([...TODAY_DISPATCH].toSorted()); } // Every protocol on both planes reaches the modern family, completions - // included (P-R1), and nothing reaches a legacy cell by default. + // included (P-R1), and nothing reaches a legacy or root cell by default. expect([...TODAY_DISPATCH].toSorted()).toEqual( PLANES.flatMap((plane) => MATRIX_PROTOCOLS.map((protocol) => `${plane} ${protocol} modern`) ).toSorted() ); expect( - [...TODAY_DISPATCH].filter((cell) => cell.endsWith("legacy")) + [...TODAY_DISPATCH].filter( + (cell) => cell.endsWith("legacy") || cell.endsWith("root") + ) ).toEqual([]); const all = new Set(keylessMatrix({ go: GO, zen: ZEN }).map(key)); for (const cell of TODAY_DISPATCH) { @@ -345,6 +405,464 @@ describe("keyless matrix · the cells the live run asserts", () => { }); }); +describe("keyless matrix · the cells endpoint: legacy dispatches to", () => { + it("LEGACY_DISPATCH is the legacy cell of every plane × protocol, but Go's Google stays on its modern cell", () => { + expect([...LEGACY_DISPATCH].toSorted()).toEqual( + [ + "opencode openai-completions legacy", + "opencode openai-responses legacy", + "opencode anthropic-messages legacy", + "opencode google-generative-ai legacy", + "opencode-go openai-completions legacy", + "opencode-go openai-responses legacy", + "opencode-go anthropic-messages legacy", + "opencode-go google-generative-ai modern", + ].toSorted() + ); + const all = new Set(keylessMatrix({ go: GO, zen: ZEN }).map(key)); + for (const cell of LEGACY_DISPATCH) { + expect(all.has(cell), cell).toBe(true); + } + // The one cell both settings share is Go's Google (`GO_GOOGLE_BASE_URL`). + expect( + [...LEGACY_DISPATCH].filter((cell) => TODAY_DISPATCH.has(cell)) + ).toEqual(["opencode-go google-generative-ai modern"]); + }); +}); + +describe("keyless matrix · the free probe", () => { + it("the free probe: every free completions model, on the default completions path, keyless, in catalog order", () => { + const cells = freeProbeCells(ZEN); + expect(cells.map((cell) => cell.model)).toEqual([ + "zen-beam", + "zen-chat-free", + ]); + for (const cell of cells) { + expect(cell.url).toBe( + "https://opencode.ai/inference/openai/v1/chat/completions" + ); + expect(cell.plane).toBe("opencode"); + expect(cell.protocol).toBe("openai-completions"); + expect(cell.family).toBe("modern"); + expect(cell.method).toBe("POST"); + expect(cell.body).toEqual({ + max_tokens: 1, + messages: [{ content: "hi", role: "user" }], + model: cell.model, + }); + expect(cell.headers).toEqual({ "content-type": "application/json" }); + } + expect(freeProbeCells([spec("paid")])).toEqual([]); + + const bundled = freeProbeCells(OPENCODE_ZEN_CATALOG); + expect(bundled.length).toBeGreaterThan(0); + for (const cell of bundled) { + const row = OPENCODE_ZEN_CATALOG.find((entry) => entry.id === cell.model); + expect(row, cell.model).toBeDefined(); + expect(row !== undefined && isFreeSpec(row), cell.model).toBe(true); + expect(protocolForSdk(row?.provider_npm), cell.model).toBe( + "openai-completions" + ); + expect( + Object.keys(cell.headers).filter((header) => + CREDENTIAL_HEADERS.has(header.toLowerCase()) + ) + ).toEqual([]); + } + }); + + it.each([ + [200, '{"choices":[]}', "served"], + [403, '{"error":{"type":"FreeTierError"}}', "free-tier-gated"], + [ + 401, + "ModelError: model x not supported for format oa-compat", + "wrong-format", + ], + [401, '{"error":{"type":"AuthError"}}', "auth-required"], + [403, "", "auth-required"], + [404, "", "path-gone"], + [410, "", "model-gone"], + [400, "Model is unavailable", "model-gone"], + [500, "", "other"], + // The vendor's withdrawal, by type at any status (the rule a turn is + // read by): never a gone path, never a shape that parsed the request. + [ + 400, + '{"type":"error","error":{"type":"ModelDeprecated","message":"Model x has been deprecated."}}', + "model-gone", + ], + [404, '{"error":{"type":"ModelDeprecated"}}', "model-gone"], + [ + 410, + '{"type":"error","error":{"type":"ModelDeprecated","message":"Model x has been deprecated."}}', + "model-gone", + ], + [400, '{"error":{"type":"ModelDeprecatedish"}}', "other"], + [404, '{"error":{"type":"ModelDeprecatedish"}}', "path-gone"], + [200, '{"choices":[{"message":{"content":"ModelDeprecated"}}]}', "served"], + // The upstream busy: the one keyless request that reaches the upstream + // is the free model's, and a busy one says nothing about the path. + [503, "", "upstream-busy"], + [529, "", "upstream-busy"], + [ + 500, + '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}', + "upstream-busy", + ], + [502, "Service Unavailable", "other"], + [400, '{"error":{"message":"max_tokens too small"}}', "other"], + [401, '{"error":{"type":"FreeTierError"}}', "auth-required"], + [302, "", "redirected"], + // The tier's usage cap is the gate's answer, not a rate limit (L3). + [ + 429, + '{"type":"error","error":{"type":"FreeUsageLimitError","message":"Free usage limit reached"}}', + "free-usage-limited", + ], + ] as const)("a free probe's %i %s → %s", (status, bodyText, verdict) => { + expect(classifyFreeProbe({ bodyText, status })).toBe(verdict); + }); + + const answer = (status: number, bodyText = "") => ({ bodyText, status }); + + it.each([ + ["served", [answer(200, '{"choices":[]}')], "pass"], + [ + "gated by the free tier", + [answer(403, '{"error":{"type":"FreeTierError"}}')], + "pass", + ], + [ + "429, then no answer (the measured rate-limit hang)", + [answer(429), undefined], + "skip", + ], + ["429, then 429", [answer(429), answer(429)], "skip"], + ["429, then served", [answer(429), answer(200)], "pass"], + // The usage cap is not retried and is not read as usable: a skip with its + // own verdict, never the pass a served model would be (L3). + [ + "the free tier's usage cap", + [answer(429, '{"error":{"type":"FreeUsageLimitError"}}')], + "skip", + ], + ["no answer, twice", [undefined, undefined], "fail"], + ["500, then 500", [answer(500), answer(500)], "fail"], + ["503, then 503 (a busy upstream)", [answer(503), answer(503)], "skip"], + ["529, then 529 (a busy upstream)", [answer(529), answer(529)], "skip"], + [ + "500 overloaded, twice", + [ + answer(500, '{"error":{"type":"overloaded_error"}}'), + answer(500, '{"error":{"type":"overloaded_error"}}'), + ], + "skip", + ], + ["503, then served", [answer(503), answer(200)], "pass"], + ["path gone", [answer(404)], "fail"], + ["a key demanded", [answer(401, '{"error":{"type":"AuthError"}}')], "fail"], + [ + "the wrong wire format", + [answer(401, "not supported for format oa-compat")], + "fail", + ], + ["a redirect", [answer(307)], "fail"], + ["the last candidate retired", [answer(410)], "fail"], + ] as const)("the free probe: %s → %s", (_name, attempts, outcome) => { + expect( + freeProbeOutcome(verdictOfAttempts(attempts, classifyFreeProbe)) + ).toBe(outcome); + }); + + it.each([ + ["one candidate served", ["served"], "served"], + ["a retired one, then one served", ["model-gone", "served"], "served"], + ["a busy one, then one served", ["upstream-busy", "served"], "served"], + [ + "a busy one, then a path gone", + ["upstream-busy", "path-gone"], + "path-gone", + ], + // Every candidate moved past: a busy one among them makes the run a + // skip, not three retired models. + [ + "busy, then two retired", + ["upstream-busy", "model-gone", "model-gone"], + "upstream-busy", + ], + [ + "three busy", + ["upstream-busy", "upstream-busy", "upstream-busy"], + "upstream-busy", + ], + ["three retired", ["model-gone", "model-gone", "model-gone"], "model-gone"], + ["no candidate sent", [], "transport-error"], + ] as const)( + "the free probe across its candidates: %s → %s", + (_name, verdicts, verdict) => { + expect(freeProbeVerdictOf(verdicts)).toBe(verdict); + } + ); + + it.each([ + ["model-gone", true], + ["upstream-busy", true], + ["served", false], + ["free-tier-gated", false], + ["path-gone", false], + ["auth-required", false], + ["wrong-format", false], + ["redirected", false], + // A 500 on a free model is the path failing, not the model: a probe + // that moved past it could pass on the next candidate's answer. + ["other", false], + ["rate-limited", false], + ["free-usage-limited", false], + ["transport-error", false], + ] as const)("the free probe moves on after %s: %s", (verdict, movesOn) => { + expect(freeProbeMovesOn(verdict)).toBe(movesOn); + }); + + describe("the free probe's candidate walk", () => { + const cells = ["free-a", "free-b", "free-c"].map((model) => ({ + body: {}, + family: "modern" as const, + headers: {}, + method: "POST" as const, + model, + plane: "opencode" as const, + protocol: "openai-completions" as const, + url: `https://example.test/${model}`, + })); + /** A sender that answers each model with its scripted attempts, recording what it sent. */ + const scripted = ( + script: Readonly> + ) => { + const sent: string[] = []; + const send = async (cell: MatrixCell) => { + sent.push(cell.model ?? ""); + return script[cell.model ?? ""] ?? []; + }; + return { send, sent }; + }; + + it("served first: one send, and that candidate decides", async () => { + const { send, sent } = scripted({ "free-a": [answer(200, "ok")] }); + const result = await runFreeProbe(cells, send); + expect(sent).toEqual(["free-a"]); + expect(result.decided).toEqual({ + bodyText: "ok", + model: "free-a", + status: 200, + verdict: "served", + }); + }); + + it("busy, then served: moves on past the busy upstream", async () => { + const { send, sent } = scripted({ + "free-a": [answer(503), answer(529)], + "free-b": [answer(200)], + }); + const result = await runFreeProbe(cells, send); + expect(sent).toEqual(["free-a", "free-b"]); + expect(result.decided.verdict).toBe("served"); + expect(result.decided.model).toBe("free-b"); + expect(result.candidates.map((candidate) => candidate.verdict)).toEqual([ + "upstream-busy", + "served", + ]); + }); + + it("busy, then rate-limited: stops on the rate limit", async () => { + const { send, sent } = scripted({ + "free-a": [answer(503), answer(503)], + "free-b": [answer(429), answer(429)], + }); + const result = await runFreeProbe(cells, send); + expect(sent).toEqual(["free-a", "free-b"]); + expect(result.decided).toMatchObject({ + model: "free-b", + status: 429, + verdict: "rate-limited", + }); + }); + + it("withdrawn at 400 (ModelDeprecated), then served: moves on past the withdrawn model", async () => { + const { send, sent } = scripted({ + "free-a": [ + answer( + 400, + '{"type":"error","error":{"type":"ModelDeprecated","message":"Model free-a has been deprecated."}}' + ), + ], + "free-b": [answer(200)], + }); + const result = await runFreeProbe(cells, send); + expect(sent).toEqual(["free-a", "free-b"]); + expect(result.candidates.map((candidate) => candidate.verdict)).toEqual([ + "model-gone", + "served", + ]); + expect(result.decided).toMatchObject({ + model: "free-b", + verdict: "served", + }); + expect(freeProbeOutcome(result.decided.verdict)).toBe("pass"); + }); + + it("served, then nothing more is sent even if a later model would retire", async () => { + const { send, sent } = scripted({ + "free-a": [answer(200)], + "free-b": [answer(410)], + }); + const result = await runFreeProbe(cells, send); + expect(sent).toEqual(["free-a"]); + expect(result.decided.verdict).toBe("served"); + }); + + it("busy, then two retired: the busy candidate decides, with its own model and status", async () => { + const { send, sent } = scripted({ + "free-a": [answer(503, "busy"), answer(503, "busy")], + "free-b": [answer(410, "free-b has been deprecated")], + "free-c": [answer(410, "free-c has been deprecated")], + }); + const result = await runFreeProbe(cells, send); + expect(sent).toEqual(["free-a", "free-b", "free-c"]); + expect(result.decided).toEqual({ + bodyText: "busy", + model: "free-a", + status: 503, + verdict: "upstream-busy", + }); + }); + + it("three retired: the last candidate decides", async () => { + const { send } = scripted({ + "free-a": [answer(410)], + "free-b": [answer(410)], + "free-c": [answer(400, "Model is unavailable")], + }); + const result = await runFreeProbe(cells, send); + expect(result.decided).toMatchObject({ + model: "free-c", + status: 400, + verdict: "model-gone", + }); + }); + + it("no candidate: nothing sent, a transport error with no model", async () => { + const { send, sent } = scripted({}); + const result = await runFreeProbe([], send); + expect(sent).toEqual([]); + expect(result.decided).toEqual({ + bodyText: "", + model: undefined, + status: undefined, + verdict: "transport-error", + }); + }); + }); + + it("the keyed and free classifiers read a busy upstream with one predicate", () => { + expect(isUpstreamBusy(503, "")).toBe(true); + expect(isUpstreamBusy(529, "")).toBe(true); + expect(isUpstreamBusy(502, '{"error":{"status":"UNAVAILABLE"}}')).toBe( + true + ); + expect(isUpstreamBusy(500, "")).toBe(false); + expect(isUpstreamBusy(429, "overloaded_error")).toBe(false); + }); +}); + +describe("keyless matrix · reading the attempts and the redirects", () => { + const answer = (status: number, bodyText = "") => ({ bodyText, status }); + + it.each([ + ["one 401", [answer(401)], "path-exists"], + ["429, then no answer", [answer(429), undefined], "rate-limited"], + ["429, then 429", [answer(429), answer(429)], "rate-limited"], + ["429, then 404", [answer(429), answer(404)], "path-gone"], + // A usage cap is the gate's answer, read by the classifier, and so is no + // answer after one: only a plain 429 is the rate limit (L3). + [ + "a usage cap", + [answer(429, '{"error":{"type":"FreeUsageLimitError"}}')], + "other", + ], + [ + "a usage cap, then no answer", + [answer(429, '{"error":{"type":"FreeUsageLimitError"}}'), undefined], + "transport-error", + ], + ["no answer, twice", [undefined, undefined], "transport-error"], + ["503, then no answer", [answer(503), undefined], "transport-error"], + ["500, then 500", [answer(500), answer(500)], "other"], + ["no attempt at all", [], "transport-error"], + ] as const)("%s → %s", (_name, attempts, verdict) => { + expect(verdictOfAttempts(attempts, classifyKeylessProbe)).toBe(verdict); + }); + + it("a redirect's path, resolved against the URL sent, without its host or query", () => { + const sent = "https://opencode.ai/zen/v1/chat/completions"; + expect(redirectPathOf("/inference/openai/v1/chat/completions", sent)).toBe( + "/inference/openai/v1/chat/completions" + ); + expect( + redirectPathOf("https://elsewhere.example/x/y?token=secret", sent) + ).toBe("/x/y"); + expect(redirectPathOf("../responses", sent)).toBe("/zen/v1/responses"); + expect(redirectPathOf(null, sent)).toBeUndefined(); + expect(redirectPathOf("", sent)).toBeUndefined(); + expect(redirectPathOf("http://[bad", sent)).toBeUndefined(); + }); +}); + +describe("keyless matrix · what the live matrix asserts", () => { + const row = (cell: string, verdict: string) => ({ + cell, + family: cell.split(" ")[2] as MatrixFamily, + verdict, + }); + + it("an unreadable answer fails on a legacy or modern cell, never on a root one", () => { + const rows = [ + row("opencode openai-completions legacy", "path-exists"), + row("opencode openai-completions root", "other"), + row("opencode openai-completions modern", "path-gone"), + row("opencode openai-responses legacy", "other"), + row("opencode openai-responses root", "transport-error"), + row("opencode openai-responses modern", "model-gone"), + row("opencode anthropic-messages legacy", "transport-error"), + row("opencode anthropic-messages root", "redirected"), + row("opencode anthropic-messages modern", "redirected"), + row("opencode google-generative-ai legacy", "rate-limited"), + row("opencode google-generative-ai modern", "no-model"), + ]; + expect(unreadRows(rows).map((entry) => entry.cell)).toEqual([ + "opencode openai-responses legacy", + "opencode anthropic-messages legacy", + "opencode anthropic-messages modern", + "opencode google-generative-ai legacy", + "opencode google-generative-ai modern", + ]); + }); + + it("only a default cell's path-gone breaks the default; a legacy or root one is recorded", () => { + const rows = [ + row("opencode openai-completions legacy", "path-gone"), + row("opencode openai-completions root", "path-gone"), + row("opencode openai-completions modern", "path-exists"), + row("opencode-go anthropic-messages modern", "path-gone"), + row("opencode-go google-generative-ai modern", "model-gone"), + ]; + expect(goneDefaultRows(rows).map((entry) => entry.cell)).toEqual([ + "opencode-go anthropic-messages modern", + ]); + expect(goneDefaultRows(rows.slice(0, 3))).toEqual([]); + }); +}); + describe("keyless matrix · reading an answer", () => { it.each([ [404, "", "path-gone"], @@ -355,7 +873,25 @@ describe("keyless matrix · reading an answer", () => { [400, "model is UNAVAILABLE for this plan", "model-gone"], [400, "Invalid API key", "path-exists"], [200, "{}", "other"], + // The vendor's withdrawal, by type at any status (the rule a turn is + // read by): never a gone path, never a shape that parsed the request. + [ + 400, + '{"type":"error","error":{"type":"ModelDeprecated","message":"Model x has been deprecated."}}', + "model-gone", + ], + [404, '{"error":{"type":"ModelDeprecated"}}', "model-gone"], + [ + 410, + '{"type":"error","error":{"type":"ModelDeprecated","message":"Model x has been deprecated."}}', + "model-gone", + ], + [400, '{"error":{"type":"ModelDeprecatedish"}}', "path-exists"], + [404, '{"error":{"type":"ModelDeprecatedish"}}', "path-gone"], + [200, '{"choices":[{"message":{"content":"ModelDeprecated"}}]}', "other"], [500, "", "other"], + [301, "", "redirected"], + [308, "", "redirected"], ] as const)("%i %s → %s", (status, bodyText, verdict) => { expect(classifyKeylessProbe({ bodyText, status })).toBe(verdict); }); @@ -394,4 +930,53 @@ describe("keyless matrix · a transient answer is retried once", () => { ] as const)("%s → %s", (status, retry) => { expect(isTransientProbeAnswer(status)).toBe(retry); }); + + it("but not a 429 naming the free tier's usage cap, which a retry meets again (L3)", () => { + const capped = '{"type":"error","error":{"type":"FreeUsageLimitError"}}'; + expect(isFreeUsageLimit(capped)).toBe(true); + expect(isTransientProbeAnswer(429, capped)).toBe(false); + expect(isTransientProbeAnswer(429, '{"error":"rate limited"}')).toBe(true); + // The name, not a word that contains it. + expect(isFreeUsageLimit("NotFreeUsageLimitErrorish")).toBe(false); + expect(isTransientProbeAnswer(500, capped)).toBe(true); + }); +}); + +describe("keyless matrix · the System One family (L13)", () => { + it("probes the Zen plane's three shapes, keyless and naming no model", () => { + const cells = systemOneCells(); + expect(cells.map((cell) => [cell.family, cell.url])).toEqual([ + ["legacy", "https://opencode.ai/zen/v1/systemone"], + ["root", "https://opencode.ai/inference/v1/systemone"], + ["modern", "https://opencode.ai/inference/systemone/v1/systemone"], + ]); + expect(Object.keys(SYSTEM_ONE_URLS).toSorted()).toEqual( + [...MATRIX_FAMILIES].toSorted() + ); + for (const cell of cells) { + expect(cell.method).toBe("POST"); + expect(cell.body).toEqual({}); + expect(cell.headers).toEqual({ "content-type": "application/json" }); + } + }); + + it("is read by the keyless classifier: a 404 is gone, an auth refusal exists", () => { + expect(classifyKeylessProbe({ bodyText: "", status: 404 })).toBe( + "path-gone" + ); + expect(classifyKeylessProbe({ bodyText: "", status: 401 })).toBe( + "path-exists" + ); + }); + + it("is no cell either setting dispatches to", () => { + for (const url of Object.values(SYSTEM_ONE_URLS)) { + for (const cell of keylessMatrix({ + go: OPENCODE_GO_CATALOG, + zen: OPENCODE_ZEN_CATALOG, + })) { + expect(cell.url.startsWith(url)).toBe(false); + } + } + }); }); diff --git a/test/mock/gateway-contract.mock.ts b/test/mock/gateway-contract.mock.ts index 09bd88c..7465cfd 100644 --- a/test/mock/gateway-contract.mock.ts +++ b/test/mock/gateway-contract.mock.ts @@ -15,11 +15,19 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + cellKey, + keylessMatrix, + LEGACY_DISPATCH, + TODAY_DISPATCH, +} from "../support/keyless-matrix.ts"; import { DEFAULT_USAGE_BASE_URL, LEGACY_PLANE_API, OPENCODE_FAMILY_API, + OPENCODE_GO_CATALOG, OPENCODE_PLANE_API, + OPENCODE_ZEN_CATALOG, } from "../support/src.ts"; import { startMockGateway, type MockGateway } from "./gateway-server.ts"; import { catalogFor, firstSpecFor, MOCK_SESSION, ZEN_KEY } from "./helpers.ts"; @@ -276,4 +284,60 @@ describe("the mock gateway", () => { "bases the plugin can send to and the mock cannot serve" ).toEqual([]); }); + + it("serves the request of every cell either endpoint setting dispatches to, and no root cell", async () => { + // The same invariant per REQUEST rather than per base: each cell the + // plugin sends a turn to under `auto`/`modern` (`TODAY_DISPATCH`) or + // `endpoint: legacy` (`LEGACY_DISPATCH`), with the protocol's own path + // and the bundled catalog's model, must reach the endpoint of its OWN + // protocol on its own plane. Reaching a route is not enough: a family + // root the mock serves answers `unknown_endpoint` for a protocol path it + // lacks, and a path could land on another protocol's endpoint — both are + // the per-protocol hole this case exists to catch (the per-base case + // above probes `/models` and cannot see it). The record's protocol + // is set before the credential check, so a cell the mock then refuses + // for its key still names the endpoint it reached; a method refusal does + // not count as served. The root shape is the converse: nothing + // dispatches there, so a mock that served it could hide a turn sent to + // the wrong family — it must answer `no_route`. + const dispatched = new Set([...TODAY_DISPATCH, ...LEGACY_DISPATCH]); + const cells = keylessMatrix({ + go: OPENCODE_GO_CATALOG, + zen: OPENCODE_ZEN_CATALOG, + }); + expect(cells.filter((cell) => dispatched.has(cellKey(cell)))).toHaveLength( + dispatched.size + ); + const unserved: string[] = []; + const servedRoot: string[] = []; + for (const cell of cells) { + const name = cellKey(cell); + if (!dispatched.has(name) && cell.family !== "root") { + continue; + } + const url = new URL(cell.url); + await fetch(`${gateway.origin}${url.pathname}${url.search}`, { + body: JSON.stringify(cell.body), + headers: { ...cell.headers, "x-opencode-session-id": MOCK_SESSION }, + method: cell.method, + }); + const record = gateway.requests.at(-1) as GatewayRequest | undefined; + if (cell.family === "root" && record?.reject !== "no_route") { + servedRoot.push(name); + } + if ( + dispatched.has(name) && + (record?.protocol !== cell.protocol || + record.plane !== cell.plane || + record.reject === "method") + ) { + unserved.push(`${name} → ${record?.reject ?? record?.protocol}`); + } + } + expect( + unserved, + "dispatched cells the mock does not serve on their own endpoint" + ).toEqual([]); + expect(servedRoot, "root cells the mock routes").toEqual([]); + }); }); diff --git a/test/probe-verdict.test.ts b/test/probe-verdict.test.ts index 0388a58..413d4ae 100644 --- a/test/probe-verdict.test.ts +++ b/test/probe-verdict.test.ts @@ -17,6 +17,9 @@ import { describe, expect, it } from "vitest"; import { accessDisabled, freeTierClosed, + freeUsageCapped, + freeWalkCappedReason, + freeWalkGap, gatewayBusy, retiredAtTheGateway, usableAsSubject, @@ -40,6 +43,15 @@ describe("the free tier's refusal", () => { freeTierClosed({ status: 403, summary: '{"type":"AuthError"}' }) ).toBe(false); }); + + it("includes the tier's usage cap, which is the same gate and not a busy gateway (L3)", () => { + const capped = { + status: 429, + summary: '{"type":"error","error":{"type":"FreeUsageLimitError"}}', + }; + expect(freeTierClosed(capped)).toBe(true); + expect(gatewayBusy(capped)).toBe(false); + }); }); describe("an account-level refusal", () => { @@ -110,6 +122,54 @@ describe("the busy gateway", () => { }); }); +describe("why a walk of free candidates found no completion", () => { + const capped = { + status: 429, + summary: '{"type":"error","error":{"type":"FreeUsageLimitError"}}', + }; + const gated = { status: 403, summary: '{"type":"FreeTierError"}' }; + const busy = { status: 429, summary: "" }; + const retired = { status: 410, summary: "" }; + + it("is the usage cap when any candidate met it and none was busy", () => { + // A walk that met the cap measured the account it spends from: reporting + // that as "the tier needs a session token" names a cause it cannot tell + // apart from the cap. The rule is ANY, not every: one capped candidate in + // a mixed walk (beside a gated and a withdrawn one) still reads as capped. + expect(freeUsageCapped(capped)).toBe(true); + expect(freeUsageCapped(gated)).toBe(false); + expect(freeWalkGap([capped, capped])).toBe("capped"); + expect(freeWalkGap([gated, capped, retired])).toBe("capped"); + }); + + it("still names a session gate the capped walk observed", () => { + // The skip reports the cap, but a mixed walk's FreeTierError is the signal + // this case exists to surface, so the reason carries it rather than + // dropping it. + expect(freeWalkCappedReason([capped, capped])).toBe( + "the free tier's usage cap (429 FreeUsageLimitError) answered, which measures the account, not the tier" + ); + expect(freeWalkCappedReason([gated, capped, retired])).toBe( + "the free tier's usage cap (429 FreeUsageLimitError) answered, which measures the account, not the tier; 1 candidate also answered 403 FreeTierError (the session gate)" + ); + expect(freeWalkCappedReason([gated, capped, gated])).toContain( + "; 2 candidates also answered 403 FreeTierError" + ); + }); + + it("is a busy gateway first, whatever else answered", () => { + expect(freeWalkGap([capped, busy])).toBe("busy"); + expect(freeWalkGap([gated, busy, retired])).toBe("busy"); + }); + + it("is a closed tier when each was gated, withdrawn or never answered", () => { + expect(freeWalkGap([gated, retired, { status: 0, summary: "" }])).toBe( + "closed" + ); + expect(freeWalkGap([])).toBe("closed"); + }); +}); + describe("a candidate is a subject when the answer was about the ROUTE", () => { it("accepts a completion, and a refusal the case can explain", () => { expect(usableAsSubject({ status: 200, summary: "ok" })).toBe(true); diff --git a/test/support/endpoint-record.ts b/test/support/endpoint-record.ts new file mode 100644 index 0000000..32efef0 --- /dev/null +++ b/test/support/endpoint-record.ts @@ -0,0 +1,276 @@ +/** + * The dated record the live probe suites write: one markdown table per suite, + * a row per plane × protocol and a cell per URL shape. + * + * `test/e2e/keyless-matrix.e2e.ts` and `test/e2e/keyed-probes.e2e.ts` print + * their tables through here, and the CI `e2e` job publishes them to the run + * summary and the `endpoint-families` artifact. `docs/redirect-planes.md` §13 + * holds the tables of a run, pasted with its date — never hand-written — and + * `test/endpoint-record.test.ts` holds that section to these columns and this + * row order. + * + * A cell is the verdict and the HTTP status, and for a redirect the path it + * pointed at; ` *` marks a cell the default + * family dispatches to (`TODAY_DISPATCH`), and ` †` one `endpoint: legacy` + * dispatches to (`LEGACY_DISPATCH`). + * + * Pure: every function is a function of its arguments. + * + * @module test/support/endpoint-record + */ +import { KEYED_VARIANTS, type KeyedVariant } from "./keyed-probes.ts"; +import { + cellKey, + LEGACY_DISPATCH, + MATRIX_FAMILIES, + MATRIX_PROTOCOLS, + type MatrixFamily, + type MatrixPlane, + type MatrixProtocol, + SYSTEM_ONE_URLS, + TODAY_DISPATCH, +} from "./keyless-matrix.ts"; + +/** One probe as the record keeps it: no header, no body. */ +export interface RecordedProbe { + readonly plane: MatrixPlane; + readonly protocol: MatrixProtocol; + readonly family: MatrixFamily; + readonly model: string | undefined; + /** `undefined` for a probe that was not answered or not sent. */ + readonly status: number | undefined; + readonly verdict: string; + /** The path a redirect pointed at: the probe never follows one. */ + readonly location?: string; +} + +/** + * A redirect target as a table cell can carry it: WHATWG keeps a `|` in a + * pathname as it is, and a raw one would split the markdown row, so it is + * percent-encoded as the URL would allow. + */ +const cellPath = (path: string): string => path.replaceAll("|", "%7C"); + +/** The record's header: plane, protocol, model, then the three shapes. */ +export const RECORD_HEADER = + "| Plane | Protocol | Model | `/zen(/go)/v1` (legacy) | `/inference(/go)/v1` (root) | `/inference(/go)/{openai,anthropic,google}/…` (modern) |"; + +/** The record's alignment row. */ +export const RECORD_RULE = "| :-- | :-- | :-- | :-- | :-- | :-- |"; + +/** The record's rows, in order: Zen then Go, each in the matrix's protocol order. */ +export const RECORD_ROWS: readonly (readonly [MatrixPlane, MatrixProtocol])[] = + (["opencode", "opencode-go"] as const).flatMap((plane) => + MATRIX_PROTOCOLS.map((protocol) => [plane, protocol] as const) + ); + +/** + * One cell of the record. + * + * @param key - the cell's {@link cellKey}. + * @param probe - what it answered, or `undefined` when it was not sent. + * @returns the verdict, its status, a redirect's target path, and the + * dispatch marks. + */ +export const recordCell = (key: string, probe?: RecordedProbe): string => { + let text = "not sent"; + if (probe !== undefined) { + const target = + probe.location === undefined ? "" : ` → ${cellPath(probe.location)}`; + text = + probe.status === undefined + ? `${probe.verdict}${target}` + : `${probe.verdict} (${probe.status}${target})`; + } + if (TODAY_DISPATCH.has(key)) { + text += " *"; + } + if (LEGACY_DISPATCH.has(key)) { + text += " †"; + } + return text; +}; + +/** + * One suite's dated table. + * + * @param record.title - which suite measured it. + * @param record.date - the UTC date of the run, `YYYY-MM-DD`. + * @param record.run - the run's {@link runReference}. + * @param record.probes - what each cell answered. + * @returns the heading, a blank line and the table, joined by newlines. + */ +export const recordTable = (record: { + readonly title: "Keyed" | "Keyless"; + readonly date: string; + readonly run: string; + readonly probes: readonly RecordedProbe[]; +}): string => { + const byKey = new Map(record.probes.map((probe) => [cellKey(probe), probe])); + const rows = RECORD_ROWS.map(([plane, protocol]) => { + const keys = MATRIX_FAMILIES.map((family) => + cellKey({ family, plane, protocol }) + ); + const model = keys + .map((key) => byKey.get(key)?.model) + .find((id) => id !== undefined); + const cells = keys.map((key) => recordCell(key, byKey.get(key))); + return `| \`${plane}\` | \`${protocol}\` | ${model === undefined ? "—" : `\`${model}\``} | ${cells.join(" | ")} |`; + }); + return [ + `### ${record.title} · measured ${record.date} (UTC) · ${record.run}`, + "", + RECORD_HEADER, + RECORD_RULE, + ...rows, + ].join("\n"); +}; + +/** The free probe's one line. */ +export const freeProbeLine = (probe: { + readonly model: string | undefined; + readonly status: number | undefined; + readonly verdict: string; +}): string => + `Free model on \`/inference/openai/v1/chat/completions\`: ${ + probe.model === undefined ? "no model" : `\`${probe.model}\`` + } → ${probe.verdict}${probe.status === undefined ? "" : ` (${probe.status})`}.`; + +/** An answer as one record cell prints it: the verdict, then the status. */ +const answerText = (probe: { + readonly status: number | undefined; + readonly verdict: string; +}): string => + probe.status === undefined + ? probe.verdict + : `${probe.verdict} (${probe.status})`; + +/** + * The System One family's one line (lesson L13): each shape's path and what + * it answered keyless, or `not sent`. + * + * @param probes - what each shape answered. + * @returns the line. + */ +export const systemOneLine = ( + probes: readonly { + readonly family: MatrixFamily; + readonly status: number | undefined; + readonly verdict: string; + }[] +): string => + `System One on the Zen plane, keyless (nothing dispatches there): ${MATRIX_FAMILIES.map( + (family) => { + const probe = probes.find((candidate) => candidate.family === family); + return `\`${new URL(SYSTEM_ONE_URLS[family]).pathname}\` (${family}) → ${ + probe === undefined ? "not sent" : answerText(probe) + }`; + } + ).join("; ")}.`; + +/** One keyed variant as the record keeps it: no header, no body. */ +export interface RecordedVariant { + readonly plane: MatrixPlane; + readonly variant: KeyedVariant["id"]; + readonly model: string | undefined; + readonly status: number | undefined; + readonly verdict: string; +} + +/** The variants table's header. */ +export const VARIANT_HEADER = + "| Plane | Probe | Lesson | Model | `/inference(/go)/openai/v1/chat/completions` |"; + +/** The variants table's alignment row. */ +export const VARIANT_RULE = "| :-- | :-- | :-- | :-- | :-- |"; + +/** The variants table's rows, in order: Zen then Go, each in {@link KEYED_VARIANTS}' order. */ +export const VARIANT_ROWS: readonly (readonly [MatrixPlane, KeyedVariant])[] = ( + ["opencode", "opencode-go"] as const +).flatMap((plane) => + KEYED_VARIANTS.map((variant) => [plane, variant] as const) +); + +/** + * The keyed variants' dated table: one row per plane and variant, the + * variant's model and what the default completions cell answered. + * + * @param record.date - the UTC date of the run, `YYYY-MM-DD`. + * @param record.run - the run's {@link runReference}. + * @param record.probes - what each variant answered. + * @returns the heading, a blank line and the table, joined by newlines. + */ +export const variantTable = (record: { + readonly date: string; + readonly run: string; + readonly probes: readonly RecordedVariant[]; +}): string => { + const rows = VARIANT_ROWS.map(([plane, variant]) => { + const probe = record.probes.find( + (candidate) => + candidate.plane === plane && candidate.variant === variant.id + ); + const model = probe?.model; + return `| \`${plane}\` | ${variant.label} | ${variant.lesson} | ${ + model === undefined ? "—" : `\`${model}\`` + } | ${probe === undefined ? "not sent" : answerText(probe)} |`; + }); + return [ + `### Keyed variants · measured ${record.date} (UTC) · ${record.run}`, + "", + VARIANT_HEADER, + VARIANT_RULE, + ...rows, + ].join("\n"); +}; + +/** + * A header value as a record line can carry it: a delay in seconds or an + * HTTP date, so anything else is dropped and the rest cut to 40 characters — + * a value the gateway sends cannot break the line or the table around it. + */ +const headerValue = (value: string): string => + value.replaceAll(/[^\w ,:.+-]/gu, "").slice(0, 40); + +/** + * The `Retry-After` line (lesson L9): every keyed 429 and the delay it named, + * or that none was answered. + * + * @param entries - each 429's cell name and its `Retry-After`, `null` when + * it carried none. + * @returns the line. + */ +export const retryAfterLine = ( + entries: readonly { readonly cell: string; readonly value: string | null }[] +): string => + entries.length === 0 + ? "`Retry-After` on a keyed 429: no 429 was answered." + : `\`Retry-After\` on a keyed 429: ${entries + .map( + ({ cell, value }) => + `${cell} → ${value === null ? "absent" : `\`${headerValue(value)}\``}` + ) + .join("; ")}.`; + +/** + * The run a record came from: a link to the CI run, or "a local run" when any + * part of it is missing. + */ +export const runReference = (run: { + readonly server: string | undefined; + readonly repository: string | undefined; + readonly runId: string | undefined; +}): string => { + const { repository, runId, server } = run; + if ( + server === undefined || + server.length === 0 || + repository === undefined || + repository.length === 0 || + runId === undefined || + runId.length === 0 + ) { + return "a local run"; + } + return `[run ${runId}](${server}/${repository}/actions/runs/${runId})`; +}; diff --git a/test/support/host.ts b/test/support/host.ts index a562461..39c2c4f 100644 --- a/test/support/host.ts +++ b/test/support/host.ts @@ -710,9 +710,10 @@ export const PI_AI_DEFAULTS: Readonly< * against the hidden-route workaround; P-R1 (the plugin owns the plane * routes) flipped the fourteen `KNOWN_DEFECT`/`INTENDED_CHANGE` facets its PR * body names to `BASELINE`, so every facet here is one no later phase may - * move — except that P-R2's measured family flip, if a `modern` cell fails, - * may move the endpoint-family part of a wire facet, re-tagging each one it - * moves and naming it in its PR body. The plugin builds every model itself, + * move — except that the `fix:` that flips `AUTO_ENDPOINT_FAMILY` on P-R2's + * measured evidence (`docs/redirect-planes.md` §13) may move the + * endpoint-family part of a wire facet, re-tagging each one it moves and + * naming it in its PR body. The plugin builds every model itself, * so one table serves every cell; a cell's table differs only where the * host's own pi or pi-ai decides ({@link S0_TODAY_BY_CELL}). * diff --git a/test/support/keyed-probes.ts b/test/support/keyed-probes.ts new file mode 100644 index 0000000..498df34 --- /dev/null +++ b/test/support/keyed-probes.ts @@ -0,0 +1,818 @@ +/** + * The keyed half of the probe matrix: the same cells as the keyless one, each + * sent with the plane's own key, and the tier that decides which live suites + * may spend one. + * + * A keyed request goes past authentication, so its answer says whether the + * gateway SERVES the protocol on that path with that plane's credential — what + * a keyless 401 cannot. Each body carries the budget + * `test/e2e/protocol-routing.e2e.ts` already sends with a key (1 to 16 output + * tokens), so a probe spends a few output tokens at most. `test/e2e/keyed-probes.e2e.ts` sends the + * cells and asserts only the ones the default family dispatches to + * (`TODAY_DISPATCH`); every other verdict is recorded. + * + * A keyed cell also carries the identity the plugin sends on every turn + * ({@link pluginIdentityHeaders}): its user agent, `x-opencode-client`, + * `x-opencode-project` and an in-band session id under both session headers. + * Without them the probe would not be the plugin's traffic — the Go plane + * refuses a keyed request with no session header (`400 MissingSessionID`, + * `docs/engineering-notes.md`), which would read as "the path parsed it" and + * measure nothing. + * + * Pure: no environment read and no fetch — the tier takes the variable's + * VALUE, and a key comes in as an argument. It reaches `src` only through the + * test door (`./src.ts`), as `./keyless-matrix.ts` does, for the user agent a + * turn carries and the SDK → protocol table the L10 variant's model is chosen + * by, so the identity the live suite sends is defined here once + * ({@link PROBE_IDENTITY}) and the offline case that holds it to the real + * fetch patch checks that same value. The keyless module still has no code + * path that can set a credential; {@link keyedCell} is the one place a key is + * added to a cell. + * + * @module test/support/keyed-probes + */ +import { + type AttemptVerdict, + type MatrixCell, + type MatrixPlane, + type MatrixProtocol, + type ProbeAnswer, + LEGACY_DISPATCH, + TODAY_DISPATCH, + isFreeSpec, + isModelDeprecated, + isUpstreamBusy, + matrixUrl, + verdictOfAttempts, +} from "./keyless-matrix.ts"; +import { type CatalogModelSpec, OPENCODE_UA, protocolForSdk } from "./src.ts"; + +/** How much of the live suite may spend a key, from `OPENCODE_E2E_KEYED`. */ +export const KEYED_TIERS = ["none", "probes", "full"] as const; +export type KeyedTier = (typeof KEYED_TIERS)[number]; + +const isKeyedTier = (value: string): value is KeyedTier => + (KEYED_TIERS as readonly string[]).includes(value); + +/** + * The tier `OPENCODE_E2E_KEYED` names. Unset or blank is `full`, so a local run + * with keys arms everything, as it did before the tiers; a value it does not + * know fails the run rather than silently arming or disarming a suite. + * + * @param value - the variable's value. + * @returns the tier. + */ +export const keyedTierOf = (value: string | undefined): KeyedTier => { + const trimmed = (value ?? "").trim(); + if (trimmed.length === 0) { + return "full"; + } + if (isKeyedTier(trimmed)) { + return trimmed; + } + throw new Error( + `OPENCODE_E2E_KEYED must be one of ${KEYED_TIERS.join(", ")}; got "${value}"` + ); +}; + +/** + * Whether a tier arms a suite that needs `need`: `full` arms both, `probes` + * only the minimal probes, `none` nothing. + */ +export const isArmed = (tier: KeyedTier, need: "full" | "probes"): boolean => + tier === "full" || (tier === "probes" && need === "probes"); + +/** + * The key a suite may use: the key when its tier arms the suite and the key is + * non-empty, else `undefined` — which every keyed suite reads as "skip". + */ +export const armedKey = ( + tier: KeyedTier, + need: "full" | "probes", + key?: string +): string | undefined => + isArmed(tier, need) && key !== undefined && key.length > 0 ? key : undefined; + +/** The header a protocol reads its credential from, carrying `key`. */ +export const credentialHeaders = ( + protocol: MatrixProtocol, + key: string +): Record => { + if (protocol === "anthropic-messages") { + return { "x-api-key": key }; + } + if (protocol === "google-generative-ai") { + return { "x-goog-api-key": key }; + } + return { authorization: `Bearer ${key}` }; +}; + +/** + * The small body each endpoint is sent WITH a key: the shapes and budgets + * `test/e2e/protocol-routing.e2e.ts` sends (`BODY_FOR`), with the model named + * in the body for the three body-addressed protocols. A non-empty `messages` + * and Google's `role` keep the gateway judging the path and the model rather + * than refusing the JSON. + */ +export const keyedBody = ( + protocol: MatrixProtocol, + model: string +): Readonly> => { + if (protocol === "openai-completions") { + return { + max_tokens: 8, + messages: [{ content: "hi", role: "user" }], + model, + stream: false, + }; + } + if (protocol === "openai-responses") { + return { input: "hi", max_output_tokens: 16, model }; + } + if (protocol === "anthropic-messages") { + return { + max_tokens: 1, + messages: [{ content: "hi", role: "user" }], + model, + }; + } + return { + contents: [{ parts: [{ text: "hi" }], role: "user" }], + generationConfig: { maxOutputTokens: 8 }, + }; +}; + +/** + * The session id a keyed probe sends: the client's shape + * (`ses_<12 hex><14 base62>`) with its low twelve bits in the band the + * gateway admits (`0xFC0..0xFFE`; the mock in `test/mock/opencode-gateway.ts` + * enforces the same band), so a refusal cannot be the id's. + */ +export const PROBE_SESSION_ID = "ses_000000000ffeKeyedProbe0001"; + +/** + * The identity headers the plugin's fetch patch sets on an OpenCode plane + * (`src/fetch-patch.ts`): its user agent, the origin client, the project + * (`global` outside a workspace), and the session id under both session + * headers. + * + * One header a turn carries is left out on purpose: `x-opencode-request`, the + * turn's message id. The gateway reads it only as a per-request metric + * (`docs/engineering-notes.md`, §x-opencode-request), and its value is one + * fresh id per request, so a constant here would match a turn no better than + * its absence does. `test/keyed-probes.test.ts` names it as the one exclusion + * when it compares this set with the wire. + * + * @param userAgent - the plugin's user agent. + * @returns the headers, lower-cased. + */ +export const pluginIdentityHeaders = ( + userAgent: string +): Readonly> => ({ + "user-agent": userAgent, + "x-opencode-client": "cli", + "x-opencode-project": "global", + "x-opencode-session": PROBE_SESSION_ID, + "x-opencode-session-id": PROBE_SESSION_ID, +}); + +/** + * The identity every keyed probe carries, defined once for the live suite and + * the offline test alike: the user agent is `OPENCODE_UA`, the one the fetch + * patch stamps on a turn — not `OPENCODE_PATCH_USER_AGENT`, which signs the + * plugin's own requests (`/usage`). `test/keyed-probes.test.ts` sends a turn + * through the real `patchFetch` and holds every header here to what it put on + * the wire, so a change to this constant, or to the patch, fails offline. + */ +export const PROBE_IDENTITY: Readonly> = + pluginIdentityHeaders(OPENCODE_UA); + +/** + * A keyless cell with the plugin's identity headers, the plane's key in the + * header its protocol reads, and the keyed body. A model-less cell keeps no + * body: it is never sent. + * + * @param cell - the keyless cell. + * @param key - the plane's key. + * @param identity - the identity headers: {@link PROBE_IDENTITY} unless a + * test needs another (the offline judge's session-less control passes `{}`). + * @returns the keyed cell. + */ +export const keyedCell = ( + cell: MatrixCell, + key: string, + identity: Readonly> = PROBE_IDENTITY +): MatrixCell => ({ + ...cell, + body: + cell.model === undefined ? undefined : keyedBody(cell.protocol, cell.model), + headers: { + ...cell.headers, + ...identity, + ...credentialHeaders(cell.protocol, key), + }, +}); + +/** What a keyed probe answered. */ +export type KeyedVerdict = + | "account" + | "auth-refused" + | "model-gone" + | "other" + | "path-exists" + | "path-gone" + | "rate-limited" + | "redirected" + | "served" + | "session-refused" + | "upstream-busy" + | "wrong-protocol"; + +/** + * A 404 body that names a model rather than a path: the upstream shapes for + * an unknown or retired model (Anthropic's `not_found_error` whose message is + * `model: `, OpenAI's `model_not_found`, Gemini's `models/ is not + * found`). Keyless, the gateway refuses before it resolves a model, so a 404 + * is the path; keyed, it can forward the upstream's answer, so a 404 is read + * by its body. + * + * Anthropic's bare `not_found_error` is NOT enough: it is the type of every + * Anthropic 404, an unknown route included (`"message":"Not Found"`), so on + * its own it would read a gone path as a retired model and lose the flip + * evidence. Nor is the body compared with the id that was sent: the gateway + * may forward a dated upstream id, and a genuinely retired model would then + * read as a gone path. + */ +const MODEL_NOT_FOUND = + /"message"\s*:\s*"model: |model_not_found|models\/[^\s"']+ is not found/iu; + +/** + * Read one keyed answer. + * + * - `redirected` is a 3xx: the probe never follows one, so a moved path is + * not recorded as its target's answer. + * - `path-gone` is a 404 whose body names no model; `model-gone` a refusal of + * type `ModelDeprecated` at any status (`isModelDeprecated`), a 410, a 400 + * "Model is unavailable", or a 404 whose body names a model + * (`MODEL_NOT_FOUND`). + * - `rate-limited` is a 429, and `account` the account rather than the path: + * no funds, or a model the account may not use. Neither says anything about + * the shape. + * - `session-refused` is the request's session refused (`MissingSessionID`, a + * `FreeTierError` on a paid model): the probe did not look like the plugin, + * so it measured nothing about the path. + * - `auth-refused` is the plane's own key refused on that path, which on a + * default-family cell means a turn carrying the same key would be refused + * too — the probe carries the plugin's identity headers, so the two + * requests match. On Go that is an entry that names + * `apiKeyEnv: OPENCODE_GO_API_KEY` (README step 3): a Go entry without it + * sends the Zen key (D20), which this probe does not measure. + * - `wrong-protocol` is the path parsing the request and refusing the model's + * protocol there. + * - `path-exists` is any other 400: the path parsed the request and refused + * its content. + * - `upstream-busy` is the upstream provider overloaded (a 503, a 529, or a + * 5xx naming an overload): the gateway may have routed the request, or its + * front may have answered before reading the key, so like a rate limit it + * says nothing about the path or the key. Any other 5xx is `other`. + * + * @param answer.status - the HTTP status. + * @param answer.bodyText - the response body, as text. + * @returns the verdict. + */ +export const classifyKeyedProbe = (answer: { + readonly status: number; + readonly bodyText: string; +}): KeyedVerdict => { + const { bodyText, status } = answer; + if (status >= 300 && status < 400) { + return "redirected"; + } + if (isModelDeprecated(status, bodyText)) { + return "model-gone"; + } + if (status === 404) { + return MODEL_NOT_FOUND.test(bodyText) ? "model-gone" : "path-gone"; + } + if (status === 410) { + return "model-gone"; + } + if (status === 429) { + return "rate-limited"; + } + if ( + status === 402 || + /Insufficient account funds|Model access is disabled/iu.test(bodyText) + ) { + return "account"; + } + if ( + (status === 400 || status === 401 || status === 403) && + /MissingSessionID|missing x-opencode-session|FreeTierError/iu.test(bodyText) + ) { + return "session-refused"; + } + if (status === 401 || status === 403) { + return "auth-refused"; + } + if (status === 400) { + if (/ModelProtocolUnsupported/u.test(bodyText)) { + return "wrong-protocol"; + } + return /model is unavailable/iu.test(bodyText) + ? "model-gone" + : "path-exists"; + } + if (status >= 200 && status < 300) { + return "served"; + } + return isUpstreamBusy(status, bodyText) ? "upstream-busy" : "other"; +}; + +/** + * One keyed probe's verdict from every attempt it made: the last answer read + * by {@link classifyKeyedProbe}, except that a 429 followed by no answer — + * the measured rate-limit hang — is `rate-limited`, not a transport error + * (`verdictOfAttempts`). + * + * @param attempts - each attempt's answer, `undefined` for one that got none. + * @returns the verdict. + */ +export const keyedVerdictOf = ( + attempts: readonly (ProbeAnswer | undefined)[] +): AttemptVerdict | KeyedVerdict => + verdictOfAttempts(attempts, classifyKeyedProbe); + +/** + * The verdicts that fail a default-family cell on their own and can be the + * flip rule's evidence (`docs/redirect-planes.md` §13): the path is gone, + * refuses the protocol, or refuses the plane's own key. Each is evidence only + * when the cell a flip would send the same plane × protocol to — the one + * `endpoint: legacy` dispatches to ({@link LEGACY_DISPATCH}), when it is not + * the default cell itself — accepted the request ({@link planeOutcome}); + * otherwise a flip would fix nothing. A `model-gone` beside that acceptance is + * evidence too, though it fails nothing alone; beside a refused protocol or + * model, the account's answer is not that acceptance, as the account's gate + * answers before either is read. A red run on flip evidence is + * the evidence for P-R2b, the `fix:` that flips `AUTO_ENDPOINT_FAMILY`. + */ +export const FLIP_EVIDENCE: ReadonlySet = new Set([ + "auth-refused", + "path-gone", + "wrong-protocol", +]); + +/** + * The verdicts that, on a cell the default family dispatches to, fail the + * job: the flip evidence, and an answer that measured nothing a turn could + * use — an unreadable status (`other`), a redirect, a refused session, no + * answer on the last attempt that did not follow a 429 (`transport-error`), + * or no model to send (`no-model`). Only {@link FLIP_EVIDENCE} can be + * evidence for a flip. + */ +const BREAKS_DEFAULT: ReadonlySet = new Set([ + ...FLIP_EVIDENCE, + "no-model", + "other", + "redirected", + "session-refused", + "transport-error", +]); + +/** Whether a verdict on a default-family cell fails the job. */ +export const breaksDefault = (verdict: string): boolean => + BREAKS_DEFAULT.has(verdict); + +/** What one plane's keyed probes decide. */ +export type PlaneOutcome = + | { + readonly kind: "fail"; + readonly broken: readonly R[]; + readonly reason: string; + } + | { readonly kind: "pass" } + | { readonly kind: "skip"; readonly reason: string }; + +/** + * The answers on another shape that show it read and accepted the plane's + * key and reached the path: served, a parsed body refused for its content, or + * the account's answer. A busy upstream is not one: a 503 can come from the + * gateway's front before the key is read. + */ +const ACCEPTED: ReadonlySet = new Set([ + "account", + "path-exists", + "served", +]); + +/** + * The answers on another shape that show it accepted the model and its + * protocol: {@link ACCEPTED} without the account's answer, as the account's + * gate answers before the format check and the model's routing + * (`test/e2e/protocol-routing.e2e.ts`), so it says nothing about either. + */ +const ACCEPTED_MODEL: ReadonlySet = new Set(["path-exists", "served"]); + +/** What another shape must have answered to have accepted what `verdict`, on a default cell, refused. */ +const acceptedBeside = (verdict: string): ReadonlySet => + verdict === "wrong-protocol" || verdict === "model-gone" + ? ACCEPTED_MODEL + : ACCEPTED; + +/** + * The answers on another shape that refused the request readably — its key, + * its path, its protocol or its model — so a flip to that shape would not + * serve it either. Anything else measured nothing: a rate limit, a busy + * upstream, no answer, an unreadable one, or the account's gate beside a + * refused protocol. + */ +const REFUSED: ReadonlySet = new Set([...FLIP_EVIDENCE, "model-gone"]); + +/** The default-cell verdicts a flip of the family could fix. */ +const FLIPPABLE: ReadonlySet = new Set([ + ...FLIP_EVIDENCE, + "model-gone", +]); + +/** Why a default cell fails a plane, in the order a reason names them. */ +const BREAK_KINDS = [ + "flip", + "unreached", + "unmeasured", + "refused", + "key", + "key-unmeasured", + "unreadable", +] as const; +type BreakKind = (typeof BREAK_KINDS)[number]; + +const BREAK_REASONS: Readonly> = { + flip: "a cell the default family dispatches to refused the protocol, the plane's own key, the path or a model that the cell `endpoint: legacy` dispatches to for the same protocol accepted (flip evidence, docs/redirect-planes.md §13)", + key: "a cell the default family dispatches to refused the plane's own key, and every other shape of the protocol that read the key refused it too (not flip evidence: check the CI secret)", + "key-unmeasured": + "a cell the default family dispatches to refused the plane's own key, and no other shape of the protocol accepted or refused it (not flip evidence: nothing else measured the key)", + refused: + "a cell the default family dispatches to refused the model's protocol or is a gone path, and the cell `endpoint: legacy` dispatches to for the same protocol refused the request too, or is the default cell itself (not flip evidence: a flip would not serve it)", + unmeasured: + "a cell the default family dispatches to refused the model's protocol or is a gone path, and the cell `endpoint: legacy` dispatches to for the same protocol answered nothing that measured the request — a rate limit, a busy upstream, the account's gate, no answer or an unreadable one (not flip evidence this run: the shape a flip reaches measured nothing; re-run)", + unreadable: + "a cell the default family dispatches to gave no readable answer (not flip evidence)", + unreached: + "a cell the default family dispatches to refused the protocol, the plane's own key, the path or a model that only a shape no setting dispatches to accepted — the root, or the `/zen/go` cell of Go's Google, which stays modern under every family (not flip evidence: no flip reaches that shape)", +}; + +/** What a default cell's siblings — the other shapes of its plane × protocol — answered. */ +interface Siblings { + /** The legacy and the root cell's verdicts, those that were probed. */ + readonly answers: readonly string[]; + /** + * The verdict of the cell a flip sends the plane × protocol to, `null` when + * that is the default cell itself (Go's Google), and `undefined` when it was + * not probed. + */ + readonly flipTarget: string | null | undefined; +} + +/** + * Why one default cell fails its plane, or `undefined` when it does not. A + * sibling counts only for what it measured: acceptance for what the default + * cell refused ({@link acceptedBeside}); for a refused key, only a sibling + * that refused the key too; for a refused protocol or a gone path, the shape + * a flip reaches, which must itself have refused the request readably + * ({@link REFUSED}) for the run to say that a flip would not serve it. + * + * @param verdict - the default cell's verdict. + * @param siblings - what the other shapes of the same plane × protocol, + * which carry the same model and key, answered. + */ +const breakKindOf = ( + verdict: string, + siblings: Siblings +): BreakKind | undefined => { + if (!FLIPPABLE.has(verdict)) { + return breaksDefault(verdict) ? "unreadable" : undefined; + } + const accepted = acceptedBeside(verdict); + const { answers, flipTarget } = siblings; + if (typeof flipTarget === "string" && accepted.has(flipTarget)) { + return "flip"; + } + if (answers.some((answer) => accepted.has(answer))) { + return "unreached"; + } + switch (verdict) { + case "model-gone": { + // "Model is unavailable" is also the answer for an id sent where it + // is not served, so it is a retired model only when no other shape + // took it. + return undefined; + } + case "auth-refused": { + // A gone path, a rate limit or a busy upstream may answer before the + // key is read, so only a sibling that refused the key says anything + // about it. + return answers.includes("auth-refused") ? "key" : "key-unmeasured"; + } + default: { + // No flip moves Go's Google, so whatever its siblings answered, a flip + // would not serve it. + return flipTarget === null || REFUSED.has(flipTarget ?? "") + ? "refused" + : "unmeasured"; + } + } +}; + +/** + * The cell a flip of the default family sends a default cell's plane × + * protocol to: the one {@link LEGACY_DISPATCH} holds for it, or `undefined` + * when that is the default cell itself (Go's Google), which no flip moves. + */ +const flipTargetOf = (cell: string): string | undefined => { + const prefix = cell.replace(/ modern$/u, " "); + const target = [...LEGACY_DISPATCH].find((key) => key.startsWith(prefix)); + return target === cell ? undefined : target; +}; + +/** + * What one plane's keyed probes decide, from the cells the default family + * dispatches to (`TODAY_DISPATCH`). The legacy and root rows never fail or + * pass a plane on their own; they are read only beside a default cell of the + * same plane × protocol, which carries the same model and key, to tell the + * default family's own refusal from one every shape gives. + * + * - `fail` when a default cell's verdict {@link breaksDefault}, or it is + * `model-gone` while another shape accepted the same model + * ({@link ACCEPTED_MODEL}: the account's answer does not count). The + * reason says whether it is flip evidence, which needs the cell a flip + * reaches — the one `endpoint: legacy` dispatches to, when that is not the + * default cell itself — to have accepted the request ({@link acceptedBeside}: + * the account's answer shows a key read and a path that exists, not a + * protocol or a model served). Acceptance only on the root, or on the + * `/zen/go` cell of Go's Google (which stays modern under every family), + * still fails the plane, as no flip reaches it. Each sibling counts only + * for what it measured: a key refused on every shape that read it is the + * CI secret, and a key no other shape read measured nothing else; a + * protocol or a path the shape a flip reaches refused too is one a flip + * would not serve, and one that shape measured nothing about (a rate limit, + * a busy upstream, the account's gate, no answer) is a re-run. None of + * them is flip evidence. + * - `skip` when no default cell was `served`: every answer was the account's, + * a rate limit, a busy upstream, a retired model or a parsed-but-refused + * body, so nothing showed that the plugin's turns are served. A skip, never + * a pass. + * - `pass` otherwise: nothing broke, and at least one default cell served. + * + * @param rows - the plane's probes: each cell's key and verdict. + * @returns the outcome. + */ +export const planeOutcome = < + R extends { readonly cell: string; readonly verdict: string }, +>( + rows: readonly R[] +): PlaneOutcome => { + const defaults = rows.filter((row) => TODAY_DISPATCH.has(row.cell)); + if (defaults.length === 0) { + return { + broken: [], + kind: "fail", + reason: "no cell the default family dispatches to was probed", + }; + } + const verdictOf = new Map(rows.map((row) => [row.cell, row.verdict])); + const siblingsOf = (cell: string): Siblings => { + const answers = ["legacy", "root"] + .map((family) => verdictOf.get(cell.replace(/ modern$/u, ` ${family}`))) + .filter((verdict) => verdict !== undefined); + const target = flipTargetOf(cell); + return { + answers, + flipTarget: target === undefined ? null : verdictOf.get(target), + }; + }; + const kinds = new Set(); + const broken = defaults.filter((row) => { + const kind = breakKindOf(row.verdict, siblingsOf(row.cell)); + if (kind !== undefined) { + kinds.add(kind); + } + return kind !== undefined; + }); + if (broken.length > 0) { + return { + broken, + kind: "fail", + reason: BREAK_KINDS.filter((kind) => kinds.has(kind)) + .map((kind) => BREAK_REASONS[kind]) + .join("; "), + }; + } + if (!defaults.some((row) => row.verdict === "served")) { + return { + kind: "skip", + reason: `no cell the default family dispatches to was served (${defaults + .map((row) => row.verdict) + .join(", ")}), so nothing about the paths was measured`, + }; + } + return { kind: "pass" }; +}; + +/** + * The start of a response body, safe to print: every occurrence of the key is + * replaced before the slice, so an error that echoes it cannot leak it. + */ +export const excerpt = (bodyText: string, key: string): string => + (key.length === 0 ? bodyText : bodyText.split(key).join("")).slice( + 0, + 200 + ); + +/** + * The `x-opencode-request` value the L4 variant sends: the client's + * `msg_<12 hex><14 base62>` shape (`newOpenCodeMessageId` in + * `src/session.ts`), constant so a run is reproducible. + */ +export const PROBE_REQUEST_ID = "msg_000000000ffeKeyedProbe0001"; + +/** + * The two `x-opencode-project` values the L5 variants send beside the + * baseline's `global`: a folder name, which is what the plugin sends inside a + * workspace (`src/stream-hook.ts`), and a 40-hex digest, the shape one + * community plugin derives. Neither names a real folder. + */ +export const PROBE_PROJECTS = { + digest: "5f2b3c1d9e8a7b6c4d3e2f1a0b9c8d7e6f5a4b3c", + folder: "my-app", +} as const; + +/** One keyed variant of the default completions cell. */ +export interface KeyedVariant { + readonly id: + | "baseline" + | "project-digest" + | "project-folder" + | "reasoning-default" + | "reasoning-none" + | "request-id"; + /** The lesson in `docs/improvement-plan.md` it measures. */ + readonly lesson: string; + /** What it changes, as the record's Probe column prints it. */ + readonly label: string; + /** Which model it sends: the matrix's representative, or a reasoning one. */ + readonly model: "reasoning" | "representative"; + /** Headers set over the probe identity. */ + readonly headers?: Readonly>; + /** Fields set over the keyed body. */ + readonly body?: Readonly>; +} + +/** + * The variants the keyed suite sends on each plane's default completions + * cell, recorded and never asserted (P-R2 measures; P6 and P-R3 decide): + * + * - L4 — `x-opencode-request`. The plugin sends a minted message id on every + * turn, and one community plugin reports any value refused; the baseline + * sends none, the variant one in the client's shape. + * - L5 — `x-opencode-project`. Three claims disagree: `global` (the + * baseline), a folder name (the plugin's own default inside a workspace), + * and a 40-hex digest. + * - L10 — `reasoning_effort: "none"` on a paid completions model that + * reasons: whether the gateway takes `none`, which would let the picker + * offer Off on plain completions models. Its own baseline sends the same + * model with no effort, so the pair differs in the one field. + * + * Lesson L9, `Retry-After`, is read off every keyed 429 instead + * ({@link retryAfterOf}). + */ +export const KEYED_VARIANTS: readonly KeyedVariant[] = [ + { + id: "baseline", + label: + "the probe identity: no `x-opencode-request`, `x-opencode-project: global`", + lesson: "L4, L5", + model: "representative", + }, + { + headers: { "x-opencode-request": PROBE_REQUEST_ID }, + id: "request-id", + label: "`x-opencode-request: msg_…`", + lesson: "L4", + model: "representative", + }, + { + headers: { "x-opencode-project": PROBE_PROJECTS.folder }, + id: "project-folder", + label: "`x-opencode-project: `", + lesson: "L5", + model: "representative", + }, + { + headers: { "x-opencode-project": PROBE_PROJECTS.digest }, + id: "project-digest", + label: "`x-opencode-project: <40 hex>`", + lesson: "L5", + model: "representative", + }, + { + id: "reasoning-default", + label: "a reasoning model, no `reasoning_effort`", + lesson: "L10", + model: "reasoning", + }, + { + body: { reasoning_effort: "none" }, + id: "reasoning-none", + label: '`reasoning_effort: "none"`', + lesson: "L10", + model: "reasoning", + }, +]; + +/** + * The paid completions model the L10 pair sends: the first catalog row that + * speaks completions, reasons, is not free and is not deprecated. + * + * @param specs - the plane's catalog slice. + * @param isDeprecated - which ids the catalog retired (default: none). + * @returns the id, or `undefined` when the slice has none. + */ +export const reasoningCompletionsModel = ( + specs: readonly CatalogModelSpec[], + isDeprecated: (id: string) => boolean = () => false +): string | undefined => + specs.find( + (spec) => + protocolForSdk(spec.provider_npm) === "openai-completions" && + spec.reasoning === true && + !isFreeSpec(spec) && + !isDeprecated(spec.id) + )?.id; + +/** One variant, ready to send: a keyed cell and the variant it carries. */ +export interface VariantCell extends MatrixCell { + readonly variant: KeyedVariant; +} + +/** + * The variants' cells for one plane: each one the plane's default + * completions cell (the modern family) made keyed ({@link keyedCell}) and + * then changed by exactly the variant's headers or body fields. A variant + * with no model keeps no body: it is never sent. + * + * @param plane - the plane. + * @param models.representative - the matrix's completions model. + * @param models.reasoning - {@link reasoningCompletionsModel}'s. + * @param key - the plane's key. + * @returns one cell per {@link KEYED_VARIANTS} entry, in order. + */ +export const keyedVariantCells = ( + plane: MatrixPlane, + models: { + readonly reasoning: string | undefined; + readonly representative: string | undefined; + }, + key: string +): VariantCell[] => + KEYED_VARIANTS.map((variant) => { + const model = models[variant.model]; + const keyed = keyedCell( + { + body: undefined, + family: "modern", + headers: { "content-type": "application/json" }, + method: "POST", + model, + plane, + protocol: "openai-completions", + url: matrixUrl(plane, "openai-completions", "modern", model), + }, + key + ); + return { + ...keyed, + body: + keyed.body === undefined + ? undefined + : { ...keyed.body, ...variant.body }, + headers: { ...keyed.headers, ...variant.headers }, + variant, + }; + }); + +/** + * What a 429 said about when to come back (lesson L9): the `Retry-After` + * value of every attempt answered 429, or `null` for one that carried none. + * The host can honour a provider's delay (dsh-llm's `providerRetryAfterMs`) + * but its pi bridge never fills it; whether the gateway sends one decides + * whether P-R3 should. + * + * @param attempts - each attempt's answer, `undefined` for one that got none. + * @returns one entry per 429, in order; empty when none was answered. + */ +export const retryAfterOf = ( + attempts: readonly (ProbeAnswer | undefined)[] +): (string | null)[] => + attempts.flatMap((answer) => + answer?.status === 429 ? [answer.retryAfter ?? null] : [] + ); diff --git a/test/support/keyless-matrix.ts b/test/support/keyless-matrix.ts index be386fe..10cc73c 100644 --- a/test/support/keyless-matrix.ts +++ b/test/support/keyless-matrix.ts @@ -1,13 +1,18 @@ /** * The keyless 401-vs-404 probe matrix: which gateway paths still exist. * - * Two planes × four protocols × two endpoint families, one representative - * PAID model per plane and protocol, every request sent WITHOUT a key. The gateway + * Two planes × four protocols × three URL shapes — legacy `/zen(/go)/v1`, the + * root `/inference(/go)/v1`, and the modern per-protocol + * `/inference(/go)/{openai,anthropic,google}/…` — one representative PAID + * model per plane and protocol, every request sent WITHOUT a key. The gateway * authenticates before it routes a model, so a keyless request tells the paths * apart without spending anything: a 404 means the path is gone, a 401/403 * that it exists, a 410 or a 400 "Model is unavailable" that the model is. - * `test/e2e/keyless-matrix.e2e.ts` sends the cells; P-R2 decides the default - * family from what they answer. + * Beside the matrix, one keyless FREE-model probe on the default completions + * path ({@link freeProbeCells}) asks whether the gateway parses a free model + * there at all. `test/e2e/keyless-matrix.e2e.ts` sends both; P-R2 measures + * them in CI (`docs/redirect-planes.md` §13, pending the first run) and keeps + * the default family (`AUTO_ENDPOINT_FAMILY`). * * Pure: every function here is a function of its arguments — no fetch, no * environment, no module state — so the cell table and the classifier are @@ -25,7 +30,26 @@ import { } from "./src.ts"; export type MatrixPlane = "opencode" | "opencode-go"; -export type MatrixFamily = "legacy" | "modern"; + +/** + * The three URL shapes, in the order cells are produced: the legacy family, + * the root, and the modern family. + */ +export const MATRIX_FAMILIES = ["legacy", "root", "modern"] as const; +export type MatrixFamily = (typeof MATRIX_FAMILIES)[number]; + +/** + * The root shape: one base per plane with every protocol under it, as legacy + * is. It is the provider-level base `models.opencode.ai` declared until the + * 2026-10-16 catalog regeneration (`docs/engineering-notes.md`, "The endpoint + * chain"); the catalog now declares the modern completions base + * (`OPENCODE_PLANE_API`). Nothing the plugin sends goes there — a route-level + * `baseURL` on it selects the modern family — so it is measured only. + */ +export const INFERENCE_ROOT: Readonly> = { + opencode: "https://opencode.ai/inference/v1", + "opencode-go": "https://opencode.ai/inference/go/v1", +}; /** The four protocols, in the order cells are produced. */ export const MATRIX_PROTOCOLS = [ @@ -66,12 +90,10 @@ export const cellKey = ( * cell's `S0_TODAY` table measured (with the default endpoint), so the two * cannot drift. * - * Not covered: a user who sets `endpoint: legacy` dispatches to the LEGACY - * cells, which the matrix records for P-R2 and does not assert; P-R2 decides - * whether that configuration is guarded. Go's Google is the exception: it - * stays on `/inference/go/google/v1beta` under every family - * (`GO_GOOGLE_BASE_URL` in `src/utils/protocols.ts`), which is the modern cell - * this set already holds. + * Not covered: a user who sets `endpoint: legacy` dispatches to the cells in + * {@link LEGACY_DISPATCH}. P-R2 decided that configuration is recorded, not + * guarded, on the live gateway: its mapping is pinned offline + * (`test/endpoint-family.test.ts`), and its verdicts are the record. */ export const TODAY_DISPATCH: ReadonlySet = new Set( (["opencode", "opencode-go"] as const).flatMap((plane) => @@ -81,13 +103,43 @@ export const TODAY_DISPATCH: ReadonlySet = new Set( ) ); +/** + * The cells the plugin sends turns to with `endpoint: legacy`: the legacy cell + * of every plane and protocol, except Go's Google, which stays on + * `/inference/go/google/v1beta` under every family (`GO_GOOGLE_BASE_URL` in + * `src/utils/protocols.ts`) — the modern cell {@link TODAY_DISPATCH} holds. + * + * Recorded, not guarded: P-R2 decided the live matrix and the keyed probes + * print these cells' verdicts (marked `†` in the record). The keyless matrix + * requires each one to answer readably, like every legacy cell + * ({@link unreadRows}), but no live assertion reads a `†` cell's path-gone — + * except Go's Google, which is also a default cell ({@link TODAY_DISPATCH}) + * and is guarded as one. What guards the configuration is the offline pin of + * its mapping. + */ +export const LEGACY_DISPATCH: ReadonlySet = new Set( + (["opencode", "opencode-go"] as const).flatMap((plane) => + MATRIX_PROTOCOLS.map((protocol) => + cellKey({ + family: + plane === "opencode-go" && protocol === "google-generative-ai" + ? "modern" + : "legacy", + plane, + protocol, + }) + ) + ) +); + /** What a probe answered, as the matrix reads it. */ export type KeylessVerdict = | "model-gone" | "no-model" | "other" | "path-exists" - | "path-gone"; + | "path-gone" + | "redirected"; /** The catalog SDK each protocol is served for, for the modern family table. */ const SDK_FOR: Readonly> = { @@ -138,6 +190,10 @@ export const matrixUrl = ( ? url.replace("?alt=sse", "") : url; } + if (family === "root") { + // The root shape: like legacy, one base per plane, `?alt=sse` kept. + return `${INFERENCE_ROOT[plane]}${suffixFor(protocol, model)}`; + } const table = OPENCODE_FAMILY_API[plane === "opencode-go" ? "go" : "zen"]; return `${table[SDK_FOR[protocol]] ?? ""}${suffixFor(protocol, model)}`; }; @@ -203,14 +259,15 @@ const representative = ( )?.id; /** - * Build the sixteen cells. + * Build the twenty-four cells. * * @param catalogs.zen - the Zen plane's catalog slice. * @param catalogs.go - the Go plane's catalog slice. * @param catalogs.isDeprecated - which ids the catalog retired (default: none). * A slice from `parseModelsDevCatalog` has already dropped its deprecated * rows, so the live matrix passes none; this guards an unparsed slice. - * @returns the cells, plane by plane, protocol by protocol, legacy first. + * @returns the cells, plane by plane, protocol by protocol, then legacy, root + * and modern. */ export const keylessMatrix = (catalogs: { readonly zen: readonly CatalogModelSpec[]; @@ -226,7 +283,7 @@ export const keylessMatrix = (catalogs: { protocol, (id) => catalogs.isDeprecated?.(plane, id) ?? false ); - for (const family of ["legacy", "modern"] as const) { + for (const family of MATRIX_FAMILIES) { cells.push({ body: model === undefined ? undefined : matrixBody(protocol, model), family, @@ -243,26 +300,206 @@ export const keylessMatrix = (catalogs: { return cells; }; +/** + * The System One family on the Zen plane, one URL per shape (lesson L13): + * TypeSafe's classification protocol, which the gateway serves beside the four + * chat protocols and pi files its classifier models under + * (`typesafe-system-one`). pi 1.1.0 names the legacy URL and pi 1.5.0 the + * modern one in its own provider; the root follows the root shape. The Go + * plane carries no System One model in either pi. + * + * This plugin serves no classifier and reads no base from pi (D26), so nothing + * dispatches here: the keyless matrix sends {@link systemOneCells} and records + * what each answers, never asserts it. + */ +export const SYSTEM_ONE_URLS: Readonly> = { + legacy: "https://opencode.ai/zen/v1/systemone", + modern: "https://opencode.ai/inference/systemone/v1/systemone", + root: "https://opencode.ai/inference/v1/systemone", +}; + +/** One keyless System One probe: a path question, so it names no model. */ +export interface SystemOneCell { + readonly family: MatrixFamily; + readonly url: string; + readonly method: "POST"; + readonly headers: Readonly>; + readonly body: Readonly>; +} + +/** + * The System One cells, legacy, root and modern: an empty JSON body and no + * credential. The gateway authenticates before it reads a body, so the answer + * is the path's — read by {@link classifyKeylessProbe}, a 404 gone and a + * 401, a 403 or a 400 there — without naming a model the bundled catalog does + * not list. + * + * @returns the three cells. + */ +export const systemOneCells = (): SystemOneCell[] => + MATRIX_FAMILIES.map((family) => ({ + body: {}, + family, + headers: { "content-type": "application/json" }, + method: "POST", + url: SYSTEM_ONE_URLS[family], + })); + +/** + * Whether a body is the free tier's usage cap: `429 FreeUsageLimitError`, the + * gate refusing a free model for the caller's usage, which a newer gateway + * answers where it used to serve (lesson L3 in `docs/improvement-plan.md`). + * It is the gate's decision, not the gateway's load, so it is neither a rate + * limit nor worth a retry. + * + * @param bodyText - the response body, as text. + * @returns whether the body names the usage cap. + */ +export const isFreeUsageLimit = (bodyText: string): boolean => + /\bFreeUsageLimitError\b/u.test(bodyText); + +/** + * Whether a refusal is the gateway's `ModelDeprecated`: the vendor withdrew + * the model, at whatever status it answers it with — measured as a `400` for + * a model the catalog still listed, and held as gone at `404` and `410` too — + * the rule {@link isModelGoneMessage} and `modelGone` in + * `test/support/live-probe.ts` read a turn by (P-R1), extended to every probe + * classifier here and in `test/support/keyed-probes.ts`. Only a refusal + * (`status >= 400`) counts, so a served answer whose text happens to carry + * the word is never read as a withdrawn model; the word is bounded, so a + * longer type that merely starts with it is not either. + * + * @param status - the HTTP status. + * @param bodyText - the response body, as text. + * @returns whether the answer is a withdrawn model. + */ +export const isModelDeprecated = (status: number, bodyText: string): boolean => + status >= 400 && /\bModelDeprecated\b/u.test(bodyText); + /** * Whether a probe's answer is transient and worth one retry before it is * classified: a rate limit, a server error, or no answer at all. The live - * matrix sends sixteen requests on every pull request, and one 429 must not - * turn the job red. + * matrix sends twenty-four requests on every pull request, and one 429 must not + * turn the job red. A 429 naming the free tier's usage cap + * ({@link isFreeUsageLimit}) is not one: the gate decided, and a retry two + * seconds later meets the same cap. * * @param status - the HTTP status, or `undefined` for a transport error. + * @param bodyText - the response body, as text, when there is one. * @returns `true` to retry once. */ -export const isTransientProbeAnswer = (status: number | undefined): boolean => - status === undefined || status === 429 || status >= 500; +export const isTransientProbeAnswer = ( + status: number | undefined, + bodyText = "" +): boolean => + status === undefined || + (status === 429 && !isFreeUsageLimit(bodyText)) || + status >= 500; + +/** One answer a probe received: its status, its body, and where a 3xx pointed. */ +export interface ProbeAnswer { + readonly status: number; + readonly bodyText: string; + /** The path a 3xx's `Location` names (never its host or query), if any. */ + readonly location?: string; + /** A 429's `Retry-After` header, as sent (lesson L9). */ + readonly retryAfter?: string; +} + +/** + * The verdicts every live probe can end in whatever it classifies with: a + * rate limit the one retry did not outlast, or no answer at all. + */ +export type AttemptVerdict = "rate-limited" | "transport-error"; + +/** + * Read the attempts one probe made — one, or two after a transient answer + * ({@link isTransientProbeAnswer}) — as one verdict. + * + * The last answer decides, with two exceptions that are the gateway's state + * rather than the path's: a final 429 is `rate-limited`, and so is NO answer + * after a 429 — measured, a rate-limited gateway answers 429 and then holds + * the next connection open until the client gives up + * (`test/e2e/protocol-routing.e2e.ts`). Any other last attempt with no answer + * is `transport-error` — after no answer, a 5xx, or a busy upstream (a 503 or + * a 529) alike: only the 429 hang is a measured gateway pattern. A 429 naming + * the free tier's usage cap ({@link isFreeUsageLimit}) is neither: it is the + * gate's answer, so `classify` reads it like any other. + * + * @param attempts - each attempt's answer, `undefined` for one that got none. + * @param classify - how the suite reads an answer. + * @returns the verdict. + */ +export const verdictOfAttempts = ( + attempts: readonly (ProbeAnswer | undefined)[], + classify: (answer: ProbeAnswer) => V +): V | AttemptVerdict => { + const isRateLimit = (answer: ProbeAnswer | undefined): boolean => + answer?.status === 429 && !isFreeUsageLimit(answer.bodyText); + const last = attempts.at(-1); + if (last === undefined) { + return attempts.some(isRateLimit) ? "rate-limited" : "transport-error"; + } + return isRateLimit(last) ? "rate-limited" : classify(last); +}; + +/** + * The path a redirect points at, resolved against the URL that was sent: no + * host and no query, so a redirect that carried a token in its query cannot + * leak it into a record. + * + * @param location - the `Location` header, or `null`. + * @param url - the URL the probe sent. + * @returns the path, or `undefined` when there is none or it cannot be read. + */ +export const redirectPathOf = ( + location: string | null, + url: string +): string | undefined => { + if (location === null || location.length === 0) { + return undefined; + } + try { + return new URL(location, url).pathname; + } catch { + return undefined; + } +}; + +/** + * A 5xx that is the upstream provider being busy, not the gateway failing on + * the path: a 503 or a 529 (Anthropic's `overloaded_error`), or any 5xx whose + * body names an overload (`overloaded_error`, Gemini's `model is overloaded` + * or `UNAVAILABLE`). A request that reaches the upstream — a keyed probe, or + * the keyless free probe — sees this routinely. The gateway may have routed + * the request, or its front may have answered before reading it, so like a + * rate limit it says nothing about the path or the key — which is why the + * keyed probes never read it as a shape accepting the request. A 500, a 502 + * or an unnamed 5xx is not busy: + * protocol-routing reads a keyed 500 as "the shape does not parse", and a + * gateway that crashes on a path must still fail. + * + * @param status - the HTTP status. + * @param bodyText - the response body, as text. + * @returns whether the upstream was busy. + */ +export const isUpstreamBusy = (status: number, bodyText: string): boolean => + status === 503 || + status === 529 || + (status >= 500 && + /overloaded_error|model is overloaded|UNAVAILABLE/u.test(bodyText)); + +/** Whether a status is a redirect, which a probe sends with `redirect: "manual"` and never follows. */ +const isRedirect = (status: number): boolean => status >= 300 && status < 400; /** * Whether a live turn's failure message says the gateway withdrew the model. * - * The same two answers {@link classifyKeylessProbe} reads as `model-gone` — a - * `410`, or a `400` saying "Model is unavailable" — as they reach a turn's - * failure message rather than a raw response, plus the gateway's - * `ModelDeprecated` error type, which it now answers for a model the catalog - * still lists (measured 2026-11-02 on `gemini-3-flash`). It is a PRE-FILTER, not a + * The same answers {@link classifyKeylessProbe} reads as `model-gone` — a + * `410`, a `400` saying "Model is unavailable", or the gateway's + * `ModelDeprecated` error type ({@link isModelDeprecated}), which it now + * answers for a model the catalog still lists (measured 2026-11-02) — as they + * reach a turn's failure message rather than a raw response. It is a PRE-FILTER, not a * verdict: the gateway answers "Model is unavailable" for an id sent where it * is not served too (`docs/upstream-findings.md` §3.8), so the live adapter * blocks move past such a model only once a raw probe of the same id @@ -276,7 +513,10 @@ export const isModelGoneMessage = (message: string): boolean => /\b410\b|model is unavailable|\bModelDeprecated\b/iu.test(message); /** - * Read one keyless answer. + * Read one keyless answer. A refusal of type `ModelDeprecated` + * ({@link isModelDeprecated}) is `model-gone` at any status, read before the + * 404 arm, so a withdrawn model answered with a 404 never reads as a gone + * default path. * * @param answer.status - the HTTP status. * @param answer.bodyText - the response body, as text. @@ -286,6 +526,12 @@ export const classifyKeylessProbe = (answer: { readonly status: number; readonly bodyText: string; }): Exclude => { + if (isRedirect(answer.status)) { + return "redirected"; + } + if (isModelDeprecated(answer.status, answer.bodyText)) { + return "model-gone"; + } if (answer.status === 404) { return "path-gone"; } @@ -302,3 +548,296 @@ export const classifyKeylessProbe = (answer: { } return "other"; }; + +/** The keyless verdicts the matrix can read: the path exists, is gone, or the model is. */ +const READABLE: ReadonlySet = new Set([ + "model-gone", + "path-exists", + "path-gone", +]); + +/** + * The matrix rows whose answer cannot be read: every legacy and modern cell + * whose verdict is not one of path-exists, path-gone or model-gone — an + * unreadable status, a redirect, a rate limit, no answer, or a model-less + * cell. A root row is never one: nothing dispatches there, so whatever it + * answers is recorded and cannot fail a pull request. A legacy row IS read: + * `endpoint: legacy`'s path-gone is recorded, not asserted, but an answer the + * matrix cannot read at all is a matrix that measured nothing. + * + * @param rows - each probed cell's family and verdict. + * @returns the unreadable rows, in order. + */ +export const unreadRows = < + R extends { readonly family: MatrixFamily; readonly verdict: string }, +>( + rows: readonly R[] +): R[] => + rows.filter((row) => row.family !== "root" && !READABLE.has(row.verdict)); + +/** + * The matrix rows that break the default: a cell the default dispatches to + * (`TODAY_DISPATCH`) whose path is gone. + * + * @param rows - each probed cell's key and verdict. + * @returns the broken rows, in order. + */ +export const goneDefaultRows = < + R extends { readonly cell: string; readonly verdict: string }, +>( + rows: readonly R[] +): R[] => + rows.filter( + (row) => TODAY_DISPATCH.has(row.cell) && row.verdict === "path-gone" + ); + +/** + * The keyless free-model probe's cells: every free model the slice serves on + * completions, in catalog order, each addressed to the default completions + * path (`/inference/openai/v1/chat/completions`) with the matrix's smallest + * body and no credential. A free model is the one class the gateway parses + * without a key, so this is the one keyless request that tells whether the + * default completions path still SERVES rather than only exists. + * + * @param specs - the Zen plane's catalog slice. + * @returns the cells; the live suite sends the first few. + */ +export const freeProbeCells = ( + specs: readonly CatalogModelSpec[] +): MatrixCell[] => + specs + .filter( + (spec) => + isFreeSpec(spec) && + protocolForSdk(spec.provider_npm) === "openai-completions" + ) + .map((spec) => ({ + body: matrixBody("openai-completions", spec.id), + family: "modern", + headers: headersFor("openai-completions"), + method: "POST", + model: spec.id, + plane: "opencode", + protocol: "openai-completions", + url: matrixUrl("opencode", "openai-completions", "modern", spec.id), + })); + +/** What the free probe answered. */ +export type FreeVerdict = + | "auth-required" + | "free-tier-gated" + | "free-usage-limited" + | "model-gone" + | "other" + | "path-gone" + | "redirected" + | "served" + | "upstream-busy" + | "wrong-format"; + +/** + * Read one free-probe answer. A `FreeTierError` is the free tier refusing a + * request it parsed (a missing session or tool schema), so it passes; a + * "not supported for format" is the model sent to an endpoint that cannot + * speak its wire format; any other 401 or 403 means the gateway wanted a key + * for a model it used to serve without one. A free model is the one keyless + * request that reaches the upstream, so a busy upstream + * ({@link isUpstreamBusy}) is `upstream-busy`, as the keyed probes read it; + * any other 5xx is `other`. A `429 FreeUsageLimitError` is + * `free-usage-limited`: the gate refusing the caller's usage of the free + * tier, its own verdict rather than a rate limit (lesson L3). A refusal of + * type `ModelDeprecated` ({@link isModelDeprecated}) is `model-gone` at any + * status, as a 410 or a 400 "Model is unavailable" is: the vendor withdrew a + * model the catalog still lists, so the probe moves on to the next candidate. + * + * @param answer.status - the HTTP status. + * @param answer.bodyText - the response body, as text. + * @returns the verdict. + */ +export const classifyFreeProbe = (answer: { + readonly status: number; + readonly bodyText: string; +}): FreeVerdict => { + if (isRedirect(answer.status)) { + return "redirected"; + } + if (isModelDeprecated(answer.status, answer.bodyText)) { + return "model-gone"; + } + if (answer.status === 404) { + return "path-gone"; + } + if ( + answer.status === 410 || + (answer.status === 400 && /model is unavailable/iu.test(answer.bodyText)) + ) { + return "model-gone"; + } + if (answer.status >= 200 && answer.status < 300) { + return "served"; + } + if (answer.status === 403 && /FreeTierError/u.test(answer.bodyText)) { + return "free-tier-gated"; + } + if (answer.status === 429 && isFreeUsageLimit(answer.bodyText)) { + return "free-usage-limited"; + } + if ( + answer.status === 401 && + /not supported for format/iu.test(answer.bodyText) + ) { + return "wrong-format"; + } + if (answer.status === 401 || answer.status === 403) { + return "auth-required"; + } + return isUpstreamBusy(answer.status, answer.bodyText) + ? "upstream-busy" + : "other"; +}; + +/** + * The free-probe verdicts that move on to the next free candidate: a retired + * model, or a busy upstream — neither says anything about the path. + */ +const FREE_MOVES_ON: ReadonlySet = new Set([ + "model-gone", + "upstream-busy", +]); + +/** Whether the free probe tries the next candidate after this verdict. */ +export const freeProbeMovesOn = (verdict: string): boolean => + FREE_MOVES_ON.has(verdict); + +/** + * Which candidate decides the free probe, from every candidate's verdict in + * the order sent: the last one, which is the first that did not move on + * ({@link freeProbeMovesOn}); when every candidate moved on, the first busy + * one if any was busy — the run measured a busy upstream, not three retired + * models — else the last. `-1` when no candidate was sent. + * + * @param verdicts - each candidate's verdict, in the order sent. + * @returns the deciding candidate's index. + */ +const decidingIndex = ( + verdicts: readonly (FreeVerdict | AttemptVerdict)[] +): number => { + const last = verdicts.length - 1; + if (last >= 0 && freeProbeMovesOn(verdicts[last] ?? "")) { + const busy = verdicts.indexOf("upstream-busy"); + return busy === -1 ? last : busy; + } + return last; +}; + +/** + * The free probe's verdict from every candidate it sent, in order: the + * deciding candidate's ({@link decidingIndex}), or `transport-error` when no + * candidate was sent. + * + * @param verdicts - each candidate's verdict, in the order sent. + * @returns the verdict the probe decides on. + */ +export const freeProbeVerdictOf = ( + verdicts: readonly (FreeVerdict | AttemptVerdict)[] +): FreeVerdict | AttemptVerdict => + verdicts[decidingIndex(verdicts)] ?? "transport-error"; + +/** One free candidate as the probe read it: what the record names. */ +export interface FreeCandidate { + readonly model: string | undefined; + /** The last attempt's status, `undefined` when it got no answer. */ + readonly status: number | undefined; + /** The last attempt's body, `""` when it got no answer. */ + readonly bodyText: string; + readonly verdict: FreeVerdict | AttemptVerdict; +} + +/** + * Walk the free candidates: send each in order ({@link freeProbeCells}), read + * its attempts ({@link verdictOfAttempts}), and stop at the first that does + * not move on ({@link freeProbeMovesOn}). The candidate that decides + * ({@link freeProbeVerdictOf}) is returned with its OWN model, status and + * body, so a busy upstream decided by an earlier candidate is never recorded + * under a later one's model and status. + * + * Pure but for the injected sender, which is the live suite's retried fetch. + * + * @param cells - the candidates, in the order to send them. + * @param send - sends one cell; every attempt's answer, `undefined` for none. + * @returns every candidate sent, and the one that decides. + */ +export const runFreeProbe = async ( + cells: readonly MatrixCell[], + send: (cell: MatrixCell) => Promise +): Promise<{ + readonly candidates: readonly FreeCandidate[]; + readonly decided: FreeCandidate; +}> => { + const candidates: FreeCandidate[] = []; + for (const cell of cells) { + // oxlint-disable-next-line no-await-in-loop -- sequential by design: the next candidate is sent only when this one moved on + const attempts = await send(cell); + const answer = attempts.at(-1); + const verdict = verdictOfAttempts(attempts, classifyFreeProbe); + candidates.push({ + bodyText: answer?.bodyText ?? "", + model: cell.model, + status: answer?.status, + verdict, + }); + if (!freeProbeMovesOn(verdict)) { + break; + } + } + return { + candidates, + decided: candidates[ + decidingIndex(candidates.map((candidate) => candidate.verdict)) + ] ?? { + bodyText: "", + model: undefined, + status: undefined, + verdict: "transport-error", + }, + }; +}; + +/** The free-probe verdicts that measured the gateway's state, not the path. */ +const FREE_PROBE_SKIPS: ReadonlySet = new Set([ + "free-usage-limited", + "rate-limited", + "upstream-busy", +]); + +/** The free-probe verdicts that mean the default completions path parses a free model. */ +const FREE_PROBE_PASSES: ReadonlySet = new Set([ + "served", + "free-tier-gated", +]); + +/** + * What the free probe's verdict decides: `pass` when the default + * completions path parsed the free model (served, or gated by the free tier); + * `skip` when the gateway was rate-limiting (`rate-limited`), the upstream + * busy (`upstream-busy`), or the free tier's usage cap answered + * (`free-usage-limited`), which say nothing about the path — free models are + * the class it limits, and `test/e2e/protocol-routing.e2e.ts`'s free cases + * skip on a rate limit too. The usage cap is the one the community plugins + * misread: counting it as a pass (as a served model) says the tier serves + * the caller when it has just refused to (lesson L3). `fail` on everything + * else: a gone path, a key + * demanded, the wrong wire format, a redirect, an unreadable status, no + * answer, or three retired free models. + * + * @param verdict - the probe's verdict ({@link freeProbeVerdictOf}). + * @returns the outcome. + */ +export const freeProbeOutcome = ( + verdict: FreeVerdict | AttemptVerdict +): "fail" | "pass" | "skip" => { + if (FREE_PROBE_PASSES.has(verdict)) { + return "pass"; + } + return FREE_PROBE_SKIPS.has(verdict) ? "skip" : "fail"; +}; diff --git a/test/support/live-probe.ts b/test/support/live-probe.ts index cc5227f..e603fad 100644 --- a/test/support/live-probe.ts +++ b/test/support/live-probe.ts @@ -16,7 +16,7 @@ * * @module test/support/live-probe */ -import { OPENCODE_PATCH_USER_AGENT } from "./src.ts"; +import { OPENCODE_UA } from "./src.ts"; /** * The path each protocol uses against an OVERRIDDEN base, which is read as a @@ -87,10 +87,14 @@ export const BODY_FOR: Readonly>> = { "openai-responses": { input: "hi", max_output_tokens: 16 }, }; -/** Headers the plugin injects, so a probe looks like the traffic we actually send. */ +/** + * Headers the plugin injects, so a probe looks like the traffic we actually + * send: a turn's user agent is `OPENCODE_UA`, which the fetch patch stamps — + * `OPENCODE_PATCH_USER_AGENT` signs only the plugin's own requests. + */ export const PLUGIN_HEADERS: Readonly> = { "content-type": "application/json", - "user-agent": OPENCODE_PATCH_USER_AGENT, + "user-agent": OPENCODE_UA, "x-opencode-client": "cli", "x-opencode-project": "global", "x-opencode-session": "ses_e2e0000000000abcdefghij", diff --git a/test/support/src.ts b/test/support/src.ts index f74aff3..e17eae5 100644 --- a/test/support/src.ts +++ b/test/support/src.ts @@ -412,8 +412,13 @@ export { export type { ReasoningSpec } from "../../src/utils/model-metadata.ts"; // utils/plane-model.ts -export { baseUrlFor, planeModelFor } from "../../src/utils/plane-model.ts"; +export { + AUTO_ENDPOINT_FAMILY, + baseUrlFor, + planeModelFor, +} from "../../src/utils/plane-model.ts"; export type { + EndpointChoice, PlaneCatalogSpec, PlaneModelInput, PlanePiModel,