From 0f51091716f3111389b8c4e38b18e2f4c2c4160f Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Thu, 12 Nov 2026 17:30:42 -0800 Subject: [PATCH 1/4] Remove unsupported project Docker default Restore the native local backend by omitting advanced project defaults, remove mandatory Docker CI preparation, and retain the pinned host launcher, model configuration and credential regression checks. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...lts.test.mjs => backend-defaults.test.mjs} | 101 +++----------- .github/workflows/evals.yml | 3 +- .vally.yaml | 5 - README.md | 24 ++-- docs/checks-and-evals.md | 127 +++++------------- 5 files changed, 62 insertions(+), 198 deletions(-) rename .github/scripts/{docker-defaults.test.mjs => backend-defaults.test.mjs} (75%) diff --git a/.github/scripts/docker-defaults.test.mjs b/.github/scripts/backend-defaults.test.mjs similarity index 75% rename from .github/scripts/docker-defaults.test.mjs rename to .github/scripts/backend-defaults.test.mjs index da159da..484d6bd 100644 --- a/.github/scripts/docker-defaults.test.mjs +++ b/.github/scripts/backend-defaults.test.mjs @@ -7,7 +7,6 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { ProjectContext, discoverAndFilterEvals } from "@microsoft/vally"; import { resolveBackendSettings } from "../../node_modules/@microsoft/vally-cli/dist/backend-selection.js"; -import { DockerBackend } from "../../node_modules/@microsoft/vally/dist/backend/docker/docker-backend.js"; import { parse } from "yaml"; const root = fileURLToPath(new URL("../../", import.meta.url)); @@ -31,7 +30,13 @@ function assertDocumentedEval(projectRoot, path) { } } -test("native full-suite and explicit-spec launches inherit Docker for every retained eval", async () => { +test("the project stays within the Evals reader contract and inherits native local execution", async () => { + // Root fields accepted by the Evals plugin 0.24.6 full-engine project reader. + const projectKeys = new Set(["environments", "params", "metadata", "paths", "suites"]); + const configuration = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8")); + for (const key of Object.keys(configuration)) { + assert.ok(projectKeys.has(key), `Unsupported Evals project root field: ${key}`); + } const project = await ProjectContext.load(root); const suite = project.resolveSuite("full"); const discovery = await discoverAndFilterEvals(project, { suiteEvals: suite.evals }); @@ -45,13 +50,16 @@ test("native full-suite and explicit-spec launches inherit Docker for every reta evalDir: dirname(filePath), cliCwd: root, }); - assert.equal(settings.name, "docker", filePath); - assert.equal(settings.args["grade-network"], "none", filePath); + assert.equal(settings.name, "local", filePath); + assert.deepEqual(settings.args, {}, filePath); } const pkg = JSON.parse(readFileSync(resolve(root, "package.json"), "utf8")); assertBackendInheritance(pkg.scripts.eval); assertBackendInheritance(pkg.scripts["eval:models"]); const workflow = parse(readFileSync(resolve(root, ".github/workflows/evals.yml"), "utf8")); + assert.equal(workflow.jobs.evals.env.COPILOT_CLI_PATH, + "${{ github.workspace }}/node_modules/@github/copilot/npm-loader.js"); + assert.doesNotMatch(JSON.stringify(workflow), /docker|compose|--backend/); const launches = workflow.jobs.evals.steps.filter((step) => /npm run eval/.test(step.run ?? "")); assert.equal(launches.length, 1); assertBackendInheritance(launches[0].run); @@ -98,7 +106,7 @@ test("documented eval checks permit removed local trees but reject missing retai test("documented credential recipes expose only the selected token and restore ambient values on failure", () => { const guide = readFileSync(resolve(root, "docs/checks-and-evals.md"), "utf8"); - const section = guide.slice(guide.indexOf("### Native Docker evals")); + const section = guide.slice(guide.indexOf("### Execution and authentication")); const bash = section.match(/```sh\n([\s\S]*?)\n```/)[1]; const powershell = section.match(/```powershell\n([\s\S]*?)\n```/)[1]; const expectedBash = String.raw`( @@ -144,8 +152,8 @@ try { assert.equal(normalizeSpec(powershell), normalizeSpec(expectedPowerShell)); const paid = guide.slice(guide.indexOf("## Paid evals")); assert.equal(normalizeSpec(paid.match(/```sh\n([\s\S]*?)\n```/)[1]), normalizeSpec(expectedBash), - "the first paid run example must include Docker prerequisites and scoped authentication"); - assert.ok(paid.indexOf("### Native Docker evals") < paid.indexOf("```sh")); + "the first paid run example must include scoped authentication"); + assert.ok(paid.indexOf("### Execution and authentication") < paid.indexOf("```sh")); const env = { ...process.env, COPILOT_GITHUB_TOKEN: "synthetic-copilot", GH_TOKEN: "synthetic-gh", GITHUB_TOKEN: "synthetic-github" }; for (const exit of [0, 37]) { const sh = spawnSync("bash", ["-c", ` @@ -210,82 +218,3 @@ exit 0 assert.ifError(empty.error); assert.equal(empty.status, 0, empty.stderr); }); - -test("a missing Docker daemon rejects native preparation rather than selecting host execution", async () => { - const backend = new DockerBackend({ - cwd: root, - cli: { - contextInspect: async () => ({ name: "test-local", dockerEndpoint: "unix:///var/run/docker.sock" }), - info: async () => { throw new Error("test Docker daemon unavailable"); }, - }, - }); - await assert.rejects(backend.prepare({}, { runId: "model-free-docker-unavailable" }), - /test Docker daemon unavailable/); - await backend.shutdown(); -}); - -test("native eval and experiment commands fail closed without Docker even with a model-free local executor", (t) => { - const directory = resolve(root, ".vally", `docker-unavailable-${randomUUID()}`); - mkdirSync(directory, { recursive: true }); - t.after(() => rmSync(directory, { recursive: true, force: true })); - const spec = resolve(directory, "eval.yaml"); - writeFileSync(spec, [ - "name: model-free-docker-unavailable", - "defaults:", - " executor: mock", - " timeout: 1s", - "stimuli:", - " - name: synthetic", - " prompt: model-free", - " graders:", - " - type: completed", - "", - ].join("\n")); - const experiment = resolve(directory, "synthetic.experiment.yaml"); - writeFileSync(experiment, [ - "name: model-free-docker-unavailable", - "evals: [eval.yaml]", - "matrix:", - " timeout:", - " path: /defaults/timeout", - " values: [1s, 2s]", - "baseline:", - " timeout: 1s", - "execution:", - " workers: 1", - "", - ].join("\n")); - for (const [command, args] of [ - ["eval", [ - "eval", "--eval-spec", spec, "--work-dir", root, - "--workers", "1", "--runs", "1", "--max-retries", "0", "--shutdown-timeout", "1s", - ]], - ["experiment", ["experiment", "run", experiment, "--workers", "1"]], - ]) { - const child = spawnSync(process.execPath, [ - resolve(root, "node_modules/@microsoft/vally-cli/dist/index.js"), - ...args, - "--workspace", resolve(directory, `${command}-workspace`), - "--output-dir", resolve(directory, `${command}-outputs`), - "--require-pass", - ], { - cwd: root, - encoding: "utf8", - timeout: 20_000, - env: { - PATH: directory, - ...(process.env.SystemRoot && { SystemRoot: process.env.SystemRoot }), - HOME: directory, - USERPROFILE: directory, - TMP: directory, - TEMP: directory, - DOCKER_CONFIG: directory, - DOCKER_CONTEXT: "model-free-nonexistent-context", - VALLY_TELEMETRY_OPTOUT: "1", - }, - }); - assert.ifError(child.error); - assert.notEqual(child.status, 0, child.stdout + child.stderr); - assert.match(child.stdout + child.stderr, /docker.*ENOENT|ENOENT.*docker|Docker backend preparation/i); - } -}); diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index ed3cae3..b613718 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -15,7 +15,7 @@ jobs: copilot-requests: write env: VALLY_TELEMETRY_OPTOUT: '1' - # .vally.yaml selects Docker; the appliance supplies the executor runtime. + COPILOT_CLI_PATH: ${{ github.workspace }}/node_modules/@github/copilot/npm-loader.js steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: @@ -25,7 +25,6 @@ jobs: node-version: 24 cache: npm - run: npm ci --no-audit --no-fund - - run: test "$(docker info --format '{{.OSType}}')" = linux && docker compose version - run: npm run eval -- --suite smoke --require-pass env: GITHUB_TOKEN: ${{ github.token }} diff --git a/.vally.yaml b/.vally.yaml index 34f9ee3..28de733 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -1,9 +1,4 @@ # Vally configuration for the included plugins' evals. -defaults: - backend: - name: docker - args: - grade-network: none paths: # evals/ mirrors plugins/: one eval per skill or agent, such as # evals//skills//eval.yaml. diff --git a/README.md b/README.md index 323183f..9ab0561 100644 --- a/README.md +++ b/README.md @@ -197,26 +197,22 @@ validation, catalog consistency checks, and remote-pin checks on every pull request. **Paid evals:** first complete the -[Docker and scoped authentication setup](docs/checks-and-evals.md#native-docker-evals). +[execution and scoped authentication setup](docs/checks-and-evals.md#execution-and-authentication). Local examples use a dedicated Copilot Requests token with public-repository access for the synthetic fixtures, not a broad GitHub CLI login credential. Then `npm run eval` runs real Copilot sessions and costs AI credits. [Evals](.github/workflows/evals.yml) runs only when manually started; pull requests never trigger paid evals. -Repository evals **default to Vally's native Docker backend** in -[.vally.yaml](.vally.yaml), including `npm run eval` and the manually dispatched -eval workflow. `npm run eval -- --suite full` selects every retained case, -including the independent reviewer agent. These runs still spend AI credits -and require Docker, Compose, network access, and container-compatible -authentication. Missing Docker or image preparation fails the run; it does not -silently fall back to host execution. Vally builds and caches its version-matched -default image locally. -See [Native Docker evals and isolation limits](docs/checks-and-evals.md#native-docker-evals) -before running. The pinned Evals plugin's Run entrypoint cannot read the new -project backend default and currently rejects this configuration; use the native -Vally CLI rather than assuming plugin Run is container-isolated. Paid CI remains -manual-only. +Repository evals use **Vally's default local backend**, without an advanced +project backend setting. This keeps [.vally.yaml](.vally.yaml) within the GitHub +Copilot Evals tools' supported project configuration. Docker and Compose are not +required by this default. Execution uses the host, not a container sandbox; +review eval setup, tools, network access and credential exposure before approval. +`npm run eval -- --suite full` selects every retained case, including the +independent reviewer agent, and still spends AI credits. See +[execution and authentication](docs/checks-and-evals.md#execution-and-authentication) +for the compatibility and isolation limits. Paid CI remains manual-only. **Standard models:** `npm run eval:models` uses the native [model experiment](evals/standard-models.experiment.yaml) to test every retained diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index 5377092..6ebf4df 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -84,17 +84,16 @@ call a model or prove how a plugin behaves in a model session. ## Paid evals **These runs start real Copilot sessions and spend AI credits.** -Complete the [Docker and scoped authentication setup](#native-docker-evals) -before running any example. An existing host login or ambient token is not -container authentication isolation. +Review the [execution and scoped authentication setup](#execution-and-authentication) +before running any example. The local backend is not a container sandbox. -Docker execution readiness means every selected case finishes task execution +Execution readiness means every selected case finishes task execution and grading without execution or grading errors. A completed behavioral **FAIL** is still a valid graded result, not an infrastructure error; preserve it for follow-up rather than changing skills, graders, or thresholds to force a pass. Commands using `--require-pass`, including the manual workflow, also require passing behavioral verdicts and can therefore exit unsuccessfully -even when Docker execution and grading completed correctly. +even when execution and grading completed correctly. Skill evals use Vally's built-in `skill-invocation` grader: `required` for every positive and boundary case, and `disallowed` for negative cases. Invocation @@ -103,9 +102,8 @@ searches, path mentions, and completed answers do not count as invocation. Custom program graders independently check the resulting files, output, and side effects. -Repository evals use the Docker backend selected in -[.vally.yaml](../.vally.yaml). The appliance supplies its own executor runtime; -do not forward a host `COPILOT_CLI_PATH` into a Linux container. The independent +Repository evals use Vally's default local backend, with no project-level +backend override in [.vally.yaml](../.vally.yaml). The independent `plugin-reviewer` eval stages its actual read-only custom agent and requires delegation through the runtime's task tool. It is included in both normal discovery and the full suite, not replaced by a companion skill. @@ -122,48 +120,28 @@ install-check recipes remain covered by model-free tests. Running a real installation check still requires the user's exact cleanup confirmation and host permissions; a headless denial is a blocker, not permission to bypass them. -### Native Docker evals - -**These are real Copilot runs and still spend AI credits.** Use Vally 0.21.0's -native [`docker` backend](https://microsoft.github.io/vally/guides/running-evals-in-docker/) -through the repository's `defaults.backend` configuration. It applies to all -seven saved evals, including explicit `--eval-spec` runs, `--suite full`, and -the manual Actions workflow. The current graders are deterministic, so the -project also defaults grading to `grade-network: none`. Missing Docker, an -inaccessible daemon, or failed appliance preparation is an error, not permission -to use the local backend. - -The pinned Evals plugin's full-engine project reader rejects the -`defaults.backend` field before Run can execute. Thus plugin Run is currently -unsupported for this repository configuration, **not** an alternate Docker -launcher or a local fallback. Its Setup writer only covers paths and suites; -this advanced native setting is maintained in `.vally.yaml`. Upstream plugin -support is needed before its Run entrypoint can honor this Docker default. - -After `npm ci`, make sure `docker` is on `PATH`, your account can access a -running **Linux** Docker daemon, and `docker compose version` succeeds. The -first uncached run needs network access and enough disk space to build Vally's -default appliance. Vally builds it from the installed core package and caches -it as `vally-task:`. Its image/core versions must match exactly; -let Vally select/build the image after upgrades rather than reusing an older -registry image. No repository Dockerfile or manual image rebuild is needed. -Host npm connectivity does not prove registry connectivity inside Docker's -build environment. Resolve build-network or certificate-trust failures before -running; do not disable TLS verification. - -On restricted-registry networks, Vally 0.21.0's default builder does not inherit -host `.npmrc` files or expose an npm-registry build override. A separately -reviewed, version-matched custom image can be selected explicitly with -`--backend docker --backend-args task-image=` and the -intended role-network arguments described below. A successful run with that -image verifies native task/grade execution, **not** the unmodified default-image -bootstrap. Never tag a modified recipe with Vally's canonical default cache tag. - -The container uses an isolated Copilot home and cannot reuse the host's -interactive Copilot login. Vally automatically forwards the recognized model -authentication variables (`COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, `GITHUB_TOKEN`) -to the task when using `copilot-sdk`. Use only the credential you intend to -expose. For local evals, use a dedicated +### Execution and authentication + +The starter leaves the backend unset. Native Vally 0.21.0 selects `local` when +no CLI, eval or project backend is specified. This is an intentional compatible +default, not a Docker failure fallback. Explicit eval selections, suites and the +standard-model experiment inherit it. + +The GitHub Copilot Evals plugin 0.24.6 full-engine project reader accepts +`environments`, `params`, `metadata`, `paths` and `suites`, but rejects the +advanced project `defaults` key. Removing that key restores the supported +configuration shape. Tools still require a compatible runner, authentication +and run approval; source/schema checks do not prove a live paid execution. + +Docker, Compose, image preparation and container network settings are not +prerequisites. The evaluator, task setup, tools and graders run on the host. +Disposable workspaces and a narrowly scoped credential do not isolate host +files, other processes, network access or the credential from evaluated code. +Review all declared effects before starting a run. + +After `npm ci`, the manual CI workflow selects the repository-pinned Copilot +launcher using `COPILOT_CLI_PATH`. Local runs use the configured native runner. +Use only the authentication you intend to expose. For local evals, use a dedicated [fine-grained token with the Copilot Requests account permission](https://docs.github.com/en/copilot/how-tos/copilot-cli/set-up-copilot-cli/authenticate-copilot-cli#authenticating-with-environment-variables): choose your personal account as resource owner, **Public repositories** access, and only **Copilot Requests** under account permissions. The synthetic fixtures @@ -228,11 +206,12 @@ Both examples clear all recognized tokens before selecting the dedicated token. Bash confines changes to a subshell; PowerShell restores the original values even if input or the eval fails. Do not enable shell tracing or print these variables. Model-free recipe checks do not prove token access or actual model -authentication. The retained Docker validation used a separately approved broad -`gh` credential; it is not evidence that these narrower credentials were tested. +authentication. Earlier Docker validation used a separately approved broad +`gh` credential; it is historical evidence, not validation of these narrower +credentials or this local-default compatibility update. Its saved results are +not changed by removing the default. -This uses the eval's declared model and one trial per case. The native image -supplies its own executor runtime, distinct from the host's pinned Copilot CLI. +This uses the eval's declared model and one trial per case. To select the two `try-plugin-locally` cases instead, replace only the `npm run eval` line **inside the same Bash subshell or PowerShell `try` block** @@ -248,39 +227,6 @@ smoke` selects the smoke cases and `npm run eval -- --suite full --max-retries positive delegations and negative no-delegation case. Review its complete case/model plan and approve its additional AI-credit exposure before starting. -Native precedence is CLI backend, then eval backend, then project backend. -An explicit `--backend` replaces the project backend argument layer. When -using an independently reviewed custom image, repeat the intended role policy, -for example `--backend docker --backend-args task-image= ---backend-args grade-network=none`; do not assume project arguments survive the -explicit override. No existing eval overrides the Docker project default. - -Know the boundary before approving a run: - -- The host still runs Vally orchestration and trusted build/setup tooling. - Task execution and grading use separate containers and workspaces, even - when they share the cached image. Hidden grading files are not staged into - the task; grading receives captured task evidence. Graders within one trial - share their grade environment and must treat agent-authored files as untrusted. -- Task and grade roles run as non-root with capabilities dropped and - `no-new-privileges`. Vally also uses root-running initialization/evidence - helpers with limited capabilities; Docker is not protection against every - kernel escape or a substitute for reviewing trusted code. -- Vally's built-in role-network defaults are Docker's normal network, **not offline**. - The task needs network for Copilot and the shared marketplace environment's - pinned remote-skill fetch. `task-network=none` is only suitable for a - self-contained model-free probe. The project sets `grade-network=none` - because the current repository graders are deterministic; LLM or network-dependent - graders need their own reviewed networking and credentials. -- The task can read and use its forwarded model credential and reach allowed - network services. Custom `task-env` and `grade-env` forwards are role-specific; - never forward judge secrets to the task. Review eval setup commands, network - effects, and credentials separately from the container boundary. -- Vally exports task workspaces and releases trial resources. Check warnings - for preserved host paths or `docker-volume://` recovery locations if export - or cleanup fails. An interrupted host or daemon can leave resources behind; - do not remove other runs' containers or volumes. - ### Standard model roster [The native experiment](../evals/standard-models.experiment.yaml) is the single @@ -290,16 +236,15 @@ IDs and set `baseline.model` to one of them. Vally requires at least two variant for a single model, use `npm run eval -- --model ` instead. After `npm ci`, preview the selected evals, variants and configuration hashes -without running setup, Docker or a model: +without running setup or a model: ```sh npm run eval:models -- --dry-run ``` `npm run eval:models` executes the planned variants and spends AI credits for -each case under each model. Use the same Docker prerequisites and scoped -authentication above; the experiment inherits the project Docker backend and -offline grading default, without forwarding a host `COPILOT_CLI_PATH`. +each case under each model. Use the same scoped authentication above; the +experiment inherits the supported local-backend configuration. The two-model roster is not a guarantee of account access or a quality claim. The baseline identifies the experiment's control, not a preferred winner; the command does not invoke an extra comparison judge. Only the model varies, From 5119b6e65b5b3516e5ff838e7b1d7d049cf4f5a9 Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Thu, 12 Nov 2026 18:02:02 -0800 Subject: [PATCH 2/4] Remove obsolete Docker setup prerequisite Keep the Requirements checklist consistent with native local eval execution. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 9ab0561..c1f4387 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,6 @@ For marketplace development and evals, use this shared setup. - **GitHub Copilot CLI 1.0.85+**. - **Vally ([`@microsoft/vally-cli`](https://www.npmjs.com/package/@microsoft/vally-cli)) 0.21.0**, pinned in `package.json`. - **Git 2.28.0+**. -- For paid repository evals, **Docker with Compose and a running Linux daemon**. ## Make it yours From 451238c99eebce65cd4cb8d9e8c289936a5c6e3e Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Thu, 12 Nov 2026 18:21:32 -0800 Subject: [PATCH 3/4] Qualify compatibility against the pinned Evals reader Verify the marketplace-pinned 0.24.0 reader as well as local 0.24.6, document their shared root-key contract, and distinguish source-shape checks from live installed-tool validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/backend-defaults.test.mjs | 4 ++-- docs/checks-and-evals.md | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/scripts/backend-defaults.test.mjs b/.github/scripts/backend-defaults.test.mjs index 484d6bd..7d48674 100644 --- a/.github/scripts/backend-defaults.test.mjs +++ b/.github/scripts/backend-defaults.test.mjs @@ -30,8 +30,8 @@ function assertDocumentedEval(projectRoot, path) { } } -test("the project stays within the Evals reader contract and inherits native local execution", async () => { - // Root fields accepted by the Evals plugin 0.24.6 full-engine project reader. +test("project root fields match the checked Evals readers and inherit native local execution", async () => { + // Shared root-key contract: marketplace-pinned Evals 0.24.0 and local 0.24.6. const projectKeys = new Set(["environments", "params", "metadata", "paths", "suites"]); const configuration = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8")); for (const key of Object.keys(configuration)) { diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index 6ebf4df..e421404 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -127,11 +127,14 @@ no CLI, eval or project backend is specified. This is an intentional compatible default, not a Docker failure fallback. Explicit eval selections, suites and the standard-model experiment inherit it. -The GitHub Copilot Evals plugin 0.24.6 full-engine project reader accepts -`environments`, `params`, `metadata`, `paths` and `suites`, but rejects the -advanced project `defaults` key. Removing that key restores the supported -configuration shape. Tools still require a compatible runner, authentication -and run approval; source/schema checks do not prove a live paid execution. +This marketplace pins the GitHub Copilot Evals plugin **0.24.0**. Its +[full-engine project reader at the pinned commit](https://github.com/github/evals/blob/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/packages/plugin/dist/mcp/server.mjs), +and the separately checked locally installed **0.24.6** reader, accept +`environments`, `params`, `metadata`, `paths` and `suites`, but reject the +advanced project `defaults` key. The regression check covers this shared root-key +shape, not an installed-version or end-to-end execution test. No plugin upgrade +is included. Tools still require a compatible runner, authentication and run +approval; these source/schema checks do not prove a live paid execution. Docker, Compose, image preparation and container network settings are not prerequisites. The evaluator, task setup, tools and graders run on the host. From 868d83b3edc811556c26deca7261802cbdabad7a Mon Sep 17 00:00:00 2001 From: Ronnie Geraghty Date: Thu, 12 Nov 2026 18:37:01 -0800 Subject: [PATCH 4/4] Link and qualify historical Docker validation evidence Make the retained validation reference inspectable without presenting the PR summary as published raw results or proof of the restored local backend. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/checks-and-evals.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index e421404..ccbb410 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -209,10 +209,12 @@ Both examples clear all recognized tokens before selecting the dedicated token. Bash confines changes to a subshell; PowerShell restores the original values even if input or the eval fails. Do not enable shell tracing or print these variables. Model-free recipe checks do not prove token access or actual model -authentication. Earlier Docker validation used a separately approved broad -`gh` credential; it is historical evidence, not validation of these narrower -credentials or this local-default compatibility update. Its saved results are -not changed by removing the default. +authentication. The [historical Docker run summary](https://github.com/github/plugin-marketplace-starter/pull/17) +records a separately approved broad `gh` credential and retained local result +paths, not a published raw-results artifact. Its reported outcomes are not +independently inspectable raw evidence or validation of these narrower +credentials or this local-default compatibility update. Removing the default +does not modify those saved files. This uses the eval's declared model and one trial per case.