diff --git a/.github/scripts/docker-defaults.test.mjs b/.github/scripts/backend-defaults.test.mjs similarity index 75% rename from .github/scripts/docker-defaults.test.mjs rename to .github/scripts/backend-defaults.test.mjs index da159da..7d48674 100644 --- a/.github/scripts/docker-defaults.test.mjs +++ b/.github/scripts/backend-defaults.test.mjs @@ -7,7 +7,6 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { ProjectContext, discoverAndFilterEvals } from "@microsoft/vally"; import { resolveBackendSettings } from "../../node_modules/@microsoft/vally-cli/dist/backend-selection.js"; -import { DockerBackend } from "../../node_modules/@microsoft/vally/dist/backend/docker/docker-backend.js"; import { parse } from "yaml"; const root = fileURLToPath(new URL("../../", import.meta.url)); @@ -31,7 +30,13 @@ function assertDocumentedEval(projectRoot, path) { } } -test("native full-suite and explicit-spec launches inherit Docker for every retained eval", async () => { +test("project root fields match the checked Evals readers and inherit native local execution", async () => { + // Shared root-key contract: marketplace-pinned Evals 0.24.0 and local 0.24.6. + const projectKeys = new Set(["environments", "params", "metadata", "paths", "suites"]); + const configuration = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8")); + for (const key of Object.keys(configuration)) { + assert.ok(projectKeys.has(key), `Unsupported Evals project root field: ${key}`); + } const project = await ProjectContext.load(root); const suite = project.resolveSuite("full"); const discovery = await discoverAndFilterEvals(project, { suiteEvals: suite.evals }); @@ -45,13 +50,16 @@ test("native full-suite and explicit-spec launches inherit Docker for every reta evalDir: dirname(filePath), cliCwd: root, }); - assert.equal(settings.name, "docker", filePath); - assert.equal(settings.args["grade-network"], "none", filePath); + assert.equal(settings.name, "local", filePath); + assert.deepEqual(settings.args, {}, filePath); } const pkg = JSON.parse(readFileSync(resolve(root, "package.json"), "utf8")); assertBackendInheritance(pkg.scripts.eval); assertBackendInheritance(pkg.scripts["eval:models"]); const workflow = parse(readFileSync(resolve(root, ".github/workflows/evals.yml"), "utf8")); + assert.equal(workflow.jobs.evals.env.COPILOT_CLI_PATH, + "${{ github.workspace }}/node_modules/@github/copilot/npm-loader.js"); + assert.doesNotMatch(JSON.stringify(workflow), /docker|compose|--backend/); const launches = workflow.jobs.evals.steps.filter((step) => /npm run eval/.test(step.run ?? "")); assert.equal(launches.length, 1); assertBackendInheritance(launches[0].run); @@ -98,7 +106,7 @@ test("documented eval checks permit removed local trees but reject missing retai test("documented credential recipes expose only the selected token and restore ambient values on failure", () => { const guide = readFileSync(resolve(root, "docs/checks-and-evals.md"), "utf8"); - const section = guide.slice(guide.indexOf("### Native Docker evals")); + const section = guide.slice(guide.indexOf("### Execution and authentication")); const bash = section.match(/```sh\n([\s\S]*?)\n```/)[1]; const powershell = section.match(/```powershell\n([\s\S]*?)\n```/)[1]; const expectedBash = String.raw`( @@ -144,8 +152,8 @@ try { assert.equal(normalizeSpec(powershell), normalizeSpec(expectedPowerShell)); const paid = guide.slice(guide.indexOf("## Paid evals")); assert.equal(normalizeSpec(paid.match(/```sh\n([\s\S]*?)\n```/)[1]), normalizeSpec(expectedBash), - "the first paid run example must include Docker prerequisites and scoped authentication"); - assert.ok(paid.indexOf("### Native Docker evals") < paid.indexOf("```sh")); + "the first paid run example must include scoped authentication"); + assert.ok(paid.indexOf("### Execution and authentication") < paid.indexOf("```sh")); const env = { ...process.env, COPILOT_GITHUB_TOKEN: "synthetic-copilot", GH_TOKEN: "synthetic-gh", GITHUB_TOKEN: "synthetic-github" }; for (const exit of [0, 37]) { const sh = spawnSync("bash", ["-c", ` @@ -210,82 +218,3 @@ exit 0 assert.ifError(empty.error); assert.equal(empty.status, 0, empty.stderr); }); - -test("a missing Docker daemon rejects native preparation rather than selecting host execution", async () => { - const backend = new DockerBackend({ - cwd: root, - cli: { - contextInspect: async () => ({ name: "test-local", dockerEndpoint: "unix:///var/run/docker.sock" }), - info: async () => { throw new Error("test Docker daemon unavailable"); }, - }, - }); - await assert.rejects(backend.prepare({}, { runId: "model-free-docker-unavailable" }), - /test Docker daemon unavailable/); - await backend.shutdown(); -}); - -test("native eval and experiment commands fail closed without Docker even with a model-free local executor", (t) => { - const directory = resolve(root, ".vally", `docker-unavailable-${randomUUID()}`); - mkdirSync(directory, { recursive: true }); - t.after(() => rmSync(directory, { recursive: true, force: true })); - const spec = resolve(directory, "eval.yaml"); - writeFileSync(spec, [ - "name: model-free-docker-unavailable", - "defaults:", - " executor: mock", - " timeout: 1s", - "stimuli:", - " - name: synthetic", - " prompt: model-free", - " graders:", - " - type: completed", - "", - ].join("\n")); - const experiment = resolve(directory, "synthetic.experiment.yaml"); - writeFileSync(experiment, [ - "name: model-free-docker-unavailable", - "evals: [eval.yaml]", - "matrix:", - " timeout:", - " path: /defaults/timeout", - " values: [1s, 2s]", - "baseline:", - " timeout: 1s", - "execution:", - " workers: 1", - "", - ].join("\n")); - for (const [command, args] of [ - ["eval", [ - "eval", "--eval-spec", spec, "--work-dir", root, - "--workers", "1", "--runs", "1", "--max-retries", "0", "--shutdown-timeout", "1s", - ]], - ["experiment", ["experiment", "run", experiment, "--workers", "1"]], - ]) { - const child = spawnSync(process.execPath, [ - resolve(root, "node_modules/@microsoft/vally-cli/dist/index.js"), - ...args, - "--workspace", resolve(directory, `${command}-workspace`), - "--output-dir", resolve(directory, `${command}-outputs`), - "--require-pass", - ], { - cwd: root, - encoding: "utf8", - timeout: 20_000, - env: { - PATH: directory, - ...(process.env.SystemRoot && { SystemRoot: process.env.SystemRoot }), - HOME: directory, - USERPROFILE: directory, - TMP: directory, - TEMP: directory, - DOCKER_CONFIG: directory, - DOCKER_CONTEXT: "model-free-nonexistent-context", - VALLY_TELEMETRY_OPTOUT: "1", - }, - }); - assert.ifError(child.error); - assert.notEqual(child.status, 0, child.stdout + child.stderr); - assert.match(child.stdout + child.stderr, /docker.*ENOENT|ENOENT.*docker|Docker backend preparation/i); - } -}); diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index ed3cae3..b613718 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -15,7 +15,7 @@ jobs: copilot-requests: write env: VALLY_TELEMETRY_OPTOUT: '1' - # .vally.yaml selects Docker; the appliance supplies the executor runtime. + COPILOT_CLI_PATH: ${{ github.workspace }}/node_modules/@github/copilot/npm-loader.js steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: @@ -25,7 +25,6 @@ jobs: node-version: 24 cache: npm - run: npm ci --no-audit --no-fund - - run: test "$(docker info --format '{{.OSType}}')" = linux && docker compose version - run: npm run eval -- --suite smoke --require-pass env: GITHUB_TOKEN: ${{ github.token }} diff --git a/.vally.yaml b/.vally.yaml index 34f9ee3..28de733 100644 --- a/.vally.yaml +++ b/.vally.yaml @@ -1,9 +1,4 @@ # Vally configuration for the included plugins' evals. -defaults: - backend: - name: docker - args: - grade-network: none paths: # evals/ mirrors plugins/: one eval per skill or agent, such as # evals//skills//eval.yaml. diff --git a/README.md b/README.md index 323183f..c1f4387 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,6 @@ For marketplace development and evals, use this shared setup. - **GitHub Copilot CLI 1.0.85+**. - **Vally ([`@microsoft/vally-cli`](https://www.npmjs.com/package/@microsoft/vally-cli)) 0.21.0**, pinned in `package.json`. - **Git 2.28.0+**. -- For paid repository evals, **Docker with Compose and a running Linux daemon**. ## Make it yours @@ -197,26 +196,22 @@ validation, catalog consistency checks, and remote-pin checks on every pull request. **Paid evals:** first complete the -[Docker and scoped authentication setup](docs/checks-and-evals.md#native-docker-evals). +[execution and scoped authentication setup](docs/checks-and-evals.md#execution-and-authentication). Local examples use a dedicated Copilot Requests token with public-repository access for the synthetic fixtures, not a broad GitHub CLI login credential. Then `npm run eval` runs real Copilot sessions and costs AI credits. [Evals](.github/workflows/evals.yml) runs only when manually started; pull requests never trigger paid evals. -Repository evals **default to Vally's native Docker backend** in -[.vally.yaml](.vally.yaml), including `npm run eval` and the manually dispatched -eval workflow. `npm run eval -- --suite full` selects every retained case, -including the independent reviewer agent. These runs still spend AI credits -and require Docker, Compose, network access, and container-compatible -authentication. Missing Docker or image preparation fails the run; it does not -silently fall back to host execution. Vally builds and caches its version-matched -default image locally. -See [Native Docker evals and isolation limits](docs/checks-and-evals.md#native-docker-evals) -before running. The pinned Evals plugin's Run entrypoint cannot read the new -project backend default and currently rejects this configuration; use the native -Vally CLI rather than assuming plugin Run is container-isolated. Paid CI remains -manual-only. +Repository evals use **Vally's default local backend**, without an advanced +project backend setting. This keeps [.vally.yaml](.vally.yaml) within the GitHub +Copilot Evals tools' supported project configuration. Docker and Compose are not +required by this default. Execution uses the host, not a container sandbox; +review eval setup, tools, network access and credential exposure before approval. +`npm run eval -- --suite full` selects every retained case, including the +independent reviewer agent, and still spends AI credits. See +[execution and authentication](docs/checks-and-evals.md#execution-and-authentication) +for the compatibility and isolation limits. Paid CI remains manual-only. **Standard models:** `npm run eval:models` uses the native [model experiment](evals/standard-models.experiment.yaml) to test every retained diff --git a/docs/checks-and-evals.md b/docs/checks-and-evals.md index 5377092..ccbb410 100644 --- a/docs/checks-and-evals.md +++ b/docs/checks-and-evals.md @@ -84,17 +84,16 @@ call a model or prove how a plugin behaves in a model session. ## Paid evals **These runs start real Copilot sessions and spend AI credits.** -Complete the [Docker and scoped authentication setup](#native-docker-evals) -before running any example. An existing host login or ambient token is not -container authentication isolation. +Review the [execution and scoped authentication setup](#execution-and-authentication) +before running any example. The local backend is not a container sandbox. -Docker execution readiness means every selected case finishes task execution +Execution readiness means every selected case finishes task execution and grading without execution or grading errors. A completed behavioral **FAIL** is still a valid graded result, not an infrastructure error; preserve it for follow-up rather than changing skills, graders, or thresholds to force a pass. Commands using `--require-pass`, including the manual workflow, also require passing behavioral verdicts and can therefore exit unsuccessfully -even when Docker execution and grading completed correctly. +even when execution and grading completed correctly. Skill evals use Vally's built-in `skill-invocation` grader: `required` for every positive and boundary case, and `disallowed` for negative cases. Invocation @@ -103,9 +102,8 @@ searches, path mentions, and completed answers do not count as invocation. Custom program graders independently check the resulting files, output, and side effects. -Repository evals use the Docker backend selected in -[.vally.yaml](../.vally.yaml). The appliance supplies its own executor runtime; -do not forward a host `COPILOT_CLI_PATH` into a Linux container. The independent +Repository evals use Vally's default local backend, with no project-level +backend override in [.vally.yaml](../.vally.yaml). The independent `plugin-reviewer` eval stages its actual read-only custom agent and requires delegation through the runtime's task tool. It is included in both normal discovery and the full suite, not replaced by a companion skill. @@ -122,48 +120,31 @@ install-check recipes remain covered by model-free tests. Running a real installation check still requires the user's exact cleanup confirmation and host permissions; a headless denial is a blocker, not permission to bypass them. -### Native Docker evals - -**These are real Copilot runs and still spend AI credits.** Use Vally 0.21.0's -native [`docker` backend](https://microsoft.github.io/vally/guides/running-evals-in-docker/) -through the repository's `defaults.backend` configuration. It applies to all -seven saved evals, including explicit `--eval-spec` runs, `--suite full`, and -the manual Actions workflow. The current graders are deterministic, so the -project also defaults grading to `grade-network: none`. Missing Docker, an -inaccessible daemon, or failed appliance preparation is an error, not permission -to use the local backend. - -The pinned Evals plugin's full-engine project reader rejects the -`defaults.backend` field before Run can execute. Thus plugin Run is currently -unsupported for this repository configuration, **not** an alternate Docker -launcher or a local fallback. Its Setup writer only covers paths and suites; -this advanced native setting is maintained in `.vally.yaml`. Upstream plugin -support is needed before its Run entrypoint can honor this Docker default. - -After `npm ci`, make sure `docker` is on `PATH`, your account can access a -running **Linux** Docker daemon, and `docker compose version` succeeds. The -first uncached run needs network access and enough disk space to build Vally's -default appliance. Vally builds it from the installed core package and caches -it as `vally-task:`. Its image/core versions must match exactly; -let Vally select/build the image after upgrades rather than reusing an older -registry image. No repository Dockerfile or manual image rebuild is needed. -Host npm connectivity does not prove registry connectivity inside Docker's -build environment. Resolve build-network or certificate-trust failures before -running; do not disable TLS verification. - -On restricted-registry networks, Vally 0.21.0's default builder does not inherit -host `.npmrc` files or expose an npm-registry build override. A separately -reviewed, version-matched custom image can be selected explicitly with -`--backend docker --backend-args task-image=` and the -intended role-network arguments described below. A successful run with that -image verifies native task/grade execution, **not** the unmodified default-image -bootstrap. Never tag a modified recipe with Vally's canonical default cache tag. - -The container uses an isolated Copilot home and cannot reuse the host's -interactive Copilot login. Vally automatically forwards the recognized model -authentication variables (`COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, `GITHUB_TOKEN`) -to the task when using `copilot-sdk`. Use only the credential you intend to -expose. For local evals, use a dedicated +### Execution and authentication + +The starter leaves the backend unset. Native Vally 0.21.0 selects `local` when +no CLI, eval or project backend is specified. This is an intentional compatible +default, not a Docker failure fallback. Explicit eval selections, suites and the +standard-model experiment inherit it. + +This marketplace pins the GitHub Copilot Evals plugin **0.24.0**. Its +[full-engine project reader at the pinned commit](https://github.com/github/evals/blob/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/packages/plugin/dist/mcp/server.mjs), +and the separately checked locally installed **0.24.6** reader, accept +`environments`, `params`, `metadata`, `paths` and `suites`, but reject the +advanced project `defaults` key. The regression check covers this shared root-key +shape, not an installed-version or end-to-end execution test. No plugin upgrade +is included. Tools still require a compatible runner, authentication and run +approval; these source/schema checks do not prove a live paid execution. + +Docker, Compose, image preparation and container network settings are not +prerequisites. The evaluator, task setup, tools and graders run on the host. +Disposable workspaces and a narrowly scoped credential do not isolate host +files, other processes, network access or the credential from evaluated code. +Review all declared effects before starting a run. + +After `npm ci`, the manual CI workflow selects the repository-pinned Copilot +launcher using `COPILOT_CLI_PATH`. Local runs use the configured native runner. +Use only the authentication you intend to expose. For local evals, use a dedicated [fine-grained token with the Copilot Requests account permission](https://docs.github.com/en/copilot/how-tos/copilot-cli/set-up-copilot-cli/authenticate-copilot-cli#authenticating-with-environment-variables): choose your personal account as resource owner, **Public repositories** access, and only **Copilot Requests** under account permissions. The synthetic fixtures @@ -228,11 +209,14 @@ Both examples clear all recognized tokens before selecting the dedicated token. Bash confines changes to a subshell; PowerShell restores the original values even if input or the eval fails. Do not enable shell tracing or print these variables. Model-free recipe checks do not prove token access or actual model -authentication. The retained Docker validation used a separately approved broad -`gh` credential; it is not evidence that these narrower credentials were tested. +authentication. The [historical Docker run summary](https://github.com/github/plugin-marketplace-starter/pull/17) +records a separately approved broad `gh` credential and retained local result +paths, not a published raw-results artifact. Its reported outcomes are not +independently inspectable raw evidence or validation of these narrower +credentials or this local-default compatibility update. Removing the default +does not modify those saved files. -This uses the eval's declared model and one trial per case. The native image -supplies its own executor runtime, distinct from the host's pinned Copilot CLI. +This uses the eval's declared model and one trial per case. To select the two `try-plugin-locally` cases instead, replace only the `npm run eval` line **inside the same Bash subshell or PowerShell `try` block** @@ -248,39 +232,6 @@ smoke` selects the smoke cases and `npm run eval -- --suite full --max-retries positive delegations and negative no-delegation case. Review its complete case/model plan and approve its additional AI-credit exposure before starting. -Native precedence is CLI backend, then eval backend, then project backend. -An explicit `--backend` replaces the project backend argument layer. When -using an independently reviewed custom image, repeat the intended role policy, -for example `--backend docker --backend-args task-image= ---backend-args grade-network=none`; do not assume project arguments survive the -explicit override. No existing eval overrides the Docker project default. - -Know the boundary before approving a run: - -- The host still runs Vally orchestration and trusted build/setup tooling. - Task execution and grading use separate containers and workspaces, even - when they share the cached image. Hidden grading files are not staged into - the task; grading receives captured task evidence. Graders within one trial - share their grade environment and must treat agent-authored files as untrusted. -- Task and grade roles run as non-root with capabilities dropped and - `no-new-privileges`. Vally also uses root-running initialization/evidence - helpers with limited capabilities; Docker is not protection against every - kernel escape or a substitute for reviewing trusted code. -- Vally's built-in role-network defaults are Docker's normal network, **not offline**. - The task needs network for Copilot and the shared marketplace environment's - pinned remote-skill fetch. `task-network=none` is only suitable for a - self-contained model-free probe. The project sets `grade-network=none` - because the current repository graders are deterministic; LLM or network-dependent - graders need their own reviewed networking and credentials. -- The task can read and use its forwarded model credential and reach allowed - network services. Custom `task-env` and `grade-env` forwards are role-specific; - never forward judge secrets to the task. Review eval setup commands, network - effects, and credentials separately from the container boundary. -- Vally exports task workspaces and releases trial resources. Check warnings - for preserved host paths or `docker-volume://` recovery locations if export - or cleanup fails. An interrupted host or daemon can leave resources behind; - do not remove other runs' containers or volumes. - ### Standard model roster [The native experiment](../evals/standard-models.experiment.yaml) is the single @@ -290,16 +241,15 @@ IDs and set `baseline.model` to one of them. Vally requires at least two variant for a single model, use `npm run eval -- --model ` instead. After `npm ci`, preview the selected evals, variants and configuration hashes -without running setup, Docker or a model: +without running setup or a model: ```sh npm run eval:models -- --dry-run ``` `npm run eval:models` executes the planned variants and spends AI credits for -each case under each model. Use the same Docker prerequisites and scoped -authentication above; the experiment inherits the project Docker backend and -offline grading default, without forwarding a host `COPILOT_CLI_PATH`. +each case under each model. Use the same scoped authentication above; the +experiment inherits the supported local-backend configuration. The two-model roster is not a guarantee of account access or a quality claim. The baseline identifies the experiment's control, not a preferred winner; the command does not invoke an extra comparison judge. Only the model varies,