test: a harness that is the real host

.github/workflows/ci.yml

@@ -315,3 +315,175 @@ jobs:
           OPENCODE_E2E: "1"
           OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
           OPENCODE_GO_API_KEY: ${{ secrets.OPENCODE_GO_API_KEY }}
+
+  # The published host binary with this checkout linked into a throwaway
+  # profile: the only run that boots `dsh` itself rather than the host packages
+  # the unit suite assembles. `test/e2e/host/probe-plugin.mjs` measures the
+  # registry, the listings, validation, where each turn is sent and the save of
+  # a catalog-unknown id, and `test/e2e/host-binary.e2e.ts` asserts today's
+  # rows (`S0_TODAY`, shared with the in-process characterization).
+  #
+  # Two cells. `pinned` installs the host with npm `overrides` forcing pi,
+  # cordis and the loader to the versions `test/support/host-versions.ts`
+  # names (the published host declares the last two as tilde ranges, so a
+  # fresh install could otherwise take a newer patch) and applies the
+  # harness's own patch to pi; `floating` takes whatever pi npm resolves for the
+  # published host (newer than the harness pins), so a pi release that moves
+  # the ground shows up nightly. Pull requests run `pinned` only, and only when
+  # they touch the host seam; dispatch and the nightly run both cells.
+  #
+  # The scope is decided by a step rather than by `on.pull_request.paths`: a
+  # workflow-level path filter would silence every other job in this file.
+  # Offline: the probe answers every gateway request itself, and the keys it
+  # stores are dummies. No secret is passed.
+  e2e-host:
+    name: e2e · host binary (${{ matrix.cell }})
+    if: github.event_name != 'push'
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix:
+        cell: ${{ github.event_name == 'pull_request' && fromJSON('["pinned"]') || fromJSON('["pinned","floating"]') }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+      - name: does this change touch the host seam?
+        id: scope
+        env:
+          EVENT: ${{ github.event_name }}
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          set -euo pipefail
+          if [ "$EVENT" != "pull_request" ]; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # Its own command, so a diff that cannot be computed (a base ref
+          # that was not fetched, a deleted base branch) fails the step rather
+          # than reading as "nothing changed" and skipping the job green.
+          changed="$(git diff --name-only "origin/${BASE_REF}...HEAD")"
+          # The job's own definition and runner config are in scope: a pull
+          # request that breaks the install, the override, the patch step or
+          # the e2e config must run the job it breaks. That includes
+          # vite.config.ts: it holds the pack config that builds lib/index.mjs
+          # (the only test that loads that bundle into a real host is this
+          # one, and its externals keep cordis and schemastery the host's
+          # copies) and the test settings vitest.e2e.config.ts inherits.
+          # package.json is not (every dependency bump touches it), so a
+          # change to its `test:e2e:host` script alone is exercised only by
+          # the nightly run.
+          # Until P-R1 moves the host seam under src/host/, the modules that
+          # install the host patches and mounts whose behaviour depends on the
+          # real loader and apply order (the hidden-route mount, the listing
+          # patch and adapter decoration, the Google bridge, the fetch patch,
+          # the plane bases and the stream hook) are in scope by name. Every
+          # other row producer (key-capture.ts, models-catalog.ts, config.ts,
+          # the generated catalog) is covered on each pull request by the
+          # in-process characterization of the same S0_TODAY rows, and here by
+          # the nightly run.
+          if printf '%s\n' "$changed" \
+            | grep -Eq '^(src/host/|src/lifecycle\.ts$|src/identity\.ts$|src/responses-provider\.ts$|src/responses-routes\.ts$|src/models-discovery\.ts$|src/google-bridge\.ts$|src/fetch-patch\.ts$|src/pi-provider\.ts$|src/stream-hook\.ts$|cordis\.patch\.yml$|test/e2e/host/|test/e2e/host-binary\.e2e\.ts$|test/support/host\.ts$|test/support/host-versions\.ts$|patches/|\.github/workflows/ci\.yml$|vitest\.e2e\.config\.ts$|vite\.config\.ts$)'; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run=false" >> "$GITHUB_OUTPUT"
+            echo "no host-seam path changed; the host binary e2e does not run for this pull request"
+          fi
+      - if: steps.scope.outputs.run == 'true'
+        uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+      - if: steps.scope.outputs.run == 'true'
+        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+        with:
+          node-version: 26
+          cache: pnpm
+      - if: steps.scope.outputs.run == 'true'
+        run: pnpm install --frozen-lockfile
+      - name: the host versions this repository pins
+        id: versions
+        if: steps.scope.outputs.run == 'true'
+        run: |
+          node --experimental-strip-types --input-type=module -e "import { HOST_VERSIONS } from './test/support/host-versions.ts'; console.log('dsh=' + HOST_VERSIONS.dsh); console.log('pi=' + HOST_VERSIONS.pi); console.log('cordis=' + HOST_VERSIONS.cordis); console.log('loader=' + HOST_VERSIONS.loader)" >> "$GITHUB_OUTPUT"
+          echo "PREFIX=$RUNNER_TEMP/dsh-host" >> "$GITHUB_ENV"
+      # The pinned install forces every host version HOST_VERSIONS names, so it
+      # is cached by those versions and the patch (the rest of the published
+      # host's tree is whatever its ranges resolved on the miss that filled
+      # the cache); the floating one is the point of re-resolving, so never.
+      - name: cache the pinned host install
+        id: host-cache
+        if: steps.scope.outputs.run == 'true' && matrix.cell == 'pinned'
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ runner.temp }}/dsh-host
+          key: dsh-host-${{ matrix.cell }}-${{ steps.versions.outputs.dsh }}-${{ steps.versions.outputs.pi }}-${{ steps.versions.outputs.cordis }}-${{ steps.versions.outputs.loader }}-${{ hashFiles('patches/*.patch') }}
+      - name: install the published host into a throwaway prefix
+        if: steps.scope.outputs.run == 'true' && steps.host-cache.outputs.cache-hit != 'true'
+        env:
+          CELL: ${{ matrix.cell }}
+          DSH: ${{ steps.versions.outputs.dsh }}
+          PI: ${{ steps.versions.outputs.pi }}
+          CORDIS: ${{ steps.versions.outputs.cordis }}
+          LOADER: ${{ steps.versions.outputs.loader }}
+        run: |
+          mkdir -p "$PREFIX"
+          # The overrides must be in the prefix's own package.json BEFORE the
+          # install, or npm resolves the wrapper's ^range to the newest pi.
+          if [ "$CELL" = pinned ]; then
+            printf '{ "private": true, "overrides": { "@earendil-works/pi-ai": "%s", "@deepseek-ai/cordis": "%s", "@deepseek-ai/cordis-plugin-loader": "%s" } }\n' "$PI" "$CORDIS" "$LOADER" > "$PREFIX/package.json"
+          else
+            printf '{ "private": true }\n' > "$PREFIX/package.json"
+          fi
+          npm install --prefix "$PREFIX" --no-audit --no-fund "@deepseek-ai/dsh@$DSH"
+          # Only on a fresh install: re-applying to a cached, patched copy fails.
+          if [ "$CELL" = pinned ]; then
+            patch -p1 -d "$PREFIX/node_modules/@earendil-works/pi-ai" < "patches/@earendil-works__pi-ai@$PI.patch"
+          fi
+      - name: assert the pi, cordis and loader the host resolved
+        if: steps.scope.outputs.run == 'true'
+        env:
+          CELL: ${{ matrix.cell }}
+          PI: ${{ steps.versions.outputs.pi }}
+          CORDIS: ${{ steps.versions.outputs.cordis }}
+          LOADER: ${{ steps.versions.outputs.loader }}
+        run: |
+          resolved="$(node -p "require(process.env.PREFIX + '/node_modules/@earendil-works/pi-ai/package.json').version")"
+          echo "cell $CELL: pi $resolved"
+          copies="$(find "$PREFIX/node_modules" -path '*/@earendil-works/pi-ai/package.json' | wc -l)"
+          if [ "$copies" -ne 1 ]; then
+            echo "::error::expected one copy of pi under the host prefix, found $copies"
+            exit 1
+          fi
+          if [ "$CELL" = pinned ] && [ "$resolved" != "$PI" ]; then
+            echo "::error::the pinned cell resolved pi $resolved, not $PI"
+            exit 1
+          fi
+          # The check test/host-versions.test.ts runs for the unit install,
+          # here for the host the binary e2e boots (a cached prefix included).
+          for pair in "cordis:$CORDIS" "cordis-plugin-loader:$LOADER"; do
+            name="@deepseek-ai/${pair%%:*}"
+            want="${pair#*:}"
+            got="$(node -p "require(process.env.PREFIX + '/node_modules/$name/package.json').version")"
+            echo "cell $CELL: $name $got"
+            if [ "$CELL" = pinned ] && [ "$got" != "$want" ]; then
+              echo "::error::the pinned cell resolved $name $got, not $want"
+              exit 1
+            fi
+          done
+      - if: steps.scope.outputs.run == 'true'
+        run: pnpm run build
+      - name: boot the host binary with the probe
+        if: steps.scope.outputs.run == 'true'
+        run: pnpm run test:e2e:host --run
+        env:
+          DSH_HOST_E2E: "1"
+          DSH_HOST_PREFIX: ${{ runner.temp }}/dsh-host
+          DSH_HOST_PI_CELL: ${{ matrix.cell }}
+          DSH_HOST_PROBE_COPY: ${{ runner.temp }}/probe-${{ matrix.cell }}.json
+      - name: upload the probe's measurement
+        if: always() && steps.scope.outputs.run == 'true'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: host-probe-${{ matrix.cell }}
+          path: ${{ runner.temp }}/probe-${{ matrix.cell }}.json
+          if-no-files-found: ignore
+          retention-days: 7

AGENTS.md

@@ -43,6 +43,9 @@ Host bundle (`lib/index.mjs`), a thin `apply` barrel over small modules:
 - `src/index.ts` · `src/identity.ts` · `src/lifecycle.ts` — the public barrel, the identity constants, and the `apply()` composition (one installer per side effect, in load-bearing order). The barrel is the host's contract and nothing more: `apply`, `Config`, `inject`, `name` and the config types. Everything else is internal, and `test/public-api.test.ts` pins the list, so widening it is a reviewed change.
 - `test/support/src.ts` — the one door tests use into `src`: each name re-exported from the module that DEFINES it, never through the barrel, so a module move edits one file. `test/architecture.test.ts` enforces both halves — every name is declared in its target, and no test outside the named exemptions (`vi.mock` targets, fresh re-imports after `vi.resetModules()`, the architecture and public-API tests, the jsdom client tests, the `reset-process-state.ts` setup file) imports a `src` path, and no jsdom test loads the door.
 - `test/support/host-versions.ts` — the host the test install runs (pi, the `@deepseek-ai` cluster, cordis and its loader), named once; `test/host-versions.test.ts` checks the installed tree against it (with `pnpm-lock.yaml` as the record of what pnpm resolved, and the published peer ranges) and `test/harness-channel.test.ts` checks the declarations and the lockfile, including that `@deepseek-ai/schemastery` resolves to one copy inside the cluster's range.
+- `test/support/host.ts` — the real host, in-process, in two tiers: **A** (`mountTierA`: a bare cordis `Context`, `LlmRuntime` and `dsh-llm-pi-ai` mounted with `{ providers }`) for the `internal/config` waterfall and pi-ai's credential paths, and **B** (`bootTierB`: dsh-app-boot's `boot()` with a temporary HOME and a test bundle) for the config editor, `settings.describe`, the volatile commit and the characterization of the plugin as it ships. Its fakes (`fakeCredentials`, `fakeLaunchEnvironment`, `recordingFetch`) keep every case offline and its credentials out of the environment (a boot blanks the two keys the plugin reads directly, should the shell export them, and restores them on dispose); `S0_PROFILE` is the contract fixture, `CHARACTERIZED_PROFILE` (S0 plus a declared catalog-unknown id) the characterization's, and `S0_TODAY` the measured per-model rows, tagged per facet, that the binary e2e shares (`todayRowsFor` picks them by the apply order the probe infers from whether the plugin's decoration wrapped pi-ai's plane adapter). Tier A has no loader, so this plugin mounted there cannot find pi-ai and mounts no hidden route and no Google bridge (whose adapter class comes from the same loader entry, `loadPiAiAdapterHost`); `test/host-harness.test.ts` exercises the builder options no contract or characterization case passes, and tier B's two failure paths (an invalid first row, an exported key). Tier B's rows carry PACKAGE names through a stub module loader, never `cordis:` builtins, because the plugin finds pi-ai's entry by `options.name` (`piAiEntry` in `src/responses-provider.ts`) and a builtin-named row would drop the hidden-route mount from the characterization. It also fixes which of pi-ai and the plugin applies first (`order`), because the loader starts rows in parallel and the published binary applies the plugin first; the held row is released when the first one applies or its fiber settles, so an invalid first row fails instead of hanging the boot.
+- `test/support/keyless-matrix.ts` — the keyless 401-vs-404 probe matrix's cell table and classifier. Pure, though it lives in `test/support/` rather than `src/utils/`: a function of its arguments, with no code path that can set a credential header; `test/e2e/keyless-matrix.e2e.ts` sends the cells.
+- `test/e2e/host/` — the measurement bundle (`probe-plugin.mjs`, its manifest and patch) the `e2e-host` job links into a throwaway profile under a temporary HOME, through a relative `link:` path. `test/e2e/host-binary.e2e.ts` is its reader, gated on `DSH_HOST_E2E=1`; it installs no host itself and needs `DSH_HOST_PREFIX`, a prefix holding the installed one.
 - `patches/` — the harness's own pi patch, copied verbatim and applied by `patchedDependencies` in `pnpm-workspace.yaml`. Its hash is pinned by `test/host-versions.test.ts` and recorded in `pnpm-lock.yaml`, so never hand-edit it: re-copy it from the harness and re-run `pnpm install`.
 - `src/config.ts` · `src/config-values.ts` · `src/utils/guards.ts` — the schema and its defaults; dependency-free readers and type guards. The client bundle may import these, and it does — along with the contract and presentation modules (`usage-contract.ts`, `login-contract.ts`, `settings-copy.ts`, `settings-fields.ts`, `usage-ui.ts`, `go-limits.ts`). The rule is not a list of names: **the client may import a module that reaches nothing host-only** — no cordis context, no credential seam, no node built-in — and the bundle gate is what enforces it, by failing the build rather than by anyone remembering. An earlier version of this line named two files as the only ones allowed, which the tree had already outgrown.
 
@@ -109,6 +112,7 @@ Web client bundle (`lib/client.js`): `src/settings-page.tsx` (entry, wiring only
 vp test run            # unit suite (refuses to start below engines.node)
 pnpm run test:shuffle  # the unit suite in random order; CI runs it on Node 24
 pnpm run test:mock     # gateway mock suite: offline, no key, runs in CI
+pnpm run test:e2e:host # opt-in: the published host binary with the probe (CONTRIBUTING, "The e2e-host job")
 vp test run --coverage # with the coverage ratchet (95/90/93/95)
 vp check               # format, lint, types
 vp fmt --write         # rewrite formatting