Remove unsupported project Docker default

.github/scripts/docker-defaults.test.mjs → .github/scripts/backend-defaults.test.mjs

@@ -7,7 +7,6 @@ import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { ProjectContext, discoverAndFilterEvals } from "@microsoft/vally";
 import { resolveBackendSettings } from "../../node_modules/@microsoft/vally-cli/dist/backend-selection.js";
-import { DockerBackend } from "../../node_modules/@microsoft/vally/dist/backend/docker/docker-backend.js";
 import { parse } from "yaml";
 
 const root = fileURLToPath(new URL("../../", import.meta.url));
@@ -31,7 +30,13 @@ function assertDocumentedEval(projectRoot, path) {
   }
 }
 
-test("native full-suite and explicit-spec launches inherit Docker for every retained eval", async () => {
+test("project root fields match the checked Evals readers and inherit native local execution", async () => {
+  // Shared root-key contract: marketplace-pinned Evals 0.24.0 and local 0.24.6.
+  const projectKeys = new Set(["environments", "params", "metadata", "paths", "suites"]);
+  const configuration = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8"));
+  for (const key of Object.keys(configuration)) {
+    assert.ok(projectKeys.has(key), `Unsupported Evals project root field: ${key}`);
+  }
   const project = await ProjectContext.load(root);
   const suite = project.resolveSuite("full");
   const discovery = await discoverAndFilterEvals(project, { suiteEvals: suite.evals });
@@ -45,13 +50,16 @@ test("native full-suite and explicit-spec launches inherit Docker for every reta
       evalDir: dirname(filePath),
       cliCwd: root,
     });
-    assert.equal(settings.name, "docker", filePath);
-    assert.equal(settings.args["grade-network"], "none", filePath);
+    assert.equal(settings.name, "local", filePath);
+    assert.deepEqual(settings.args, {}, filePath);
   }
   const pkg = JSON.parse(readFileSync(resolve(root, "package.json"), "utf8"));
   assertBackendInheritance(pkg.scripts.eval);
   assertBackendInheritance(pkg.scripts["eval:models"]);
   const workflow = parse(readFileSync(resolve(root, ".github/workflows/evals.yml"), "utf8"));
+  assert.equal(workflow.jobs.evals.env.COPILOT_CLI_PATH,
+    "${{ github.workspace }}/node_modules/@github/copilot/npm-loader.js");
+  assert.doesNotMatch(JSON.stringify(workflow), /docker|compose|--backend/);
   const launches = workflow.jobs.evals.steps.filter((step) => /npm run eval/.test(step.run ?? ""));
   assert.equal(launches.length, 1);
   assertBackendInheritance(launches[0].run);
@@ -98,7 +106,7 @@ test("documented eval checks permit removed local trees but reject missing retai
 
 test("documented credential recipes expose only the selected token and restore ambient values on failure", () => {
   const guide = readFileSync(resolve(root, "docs/checks-and-evals.md"), "utf8");
-  const section = guide.slice(guide.indexOf("### Native Docker evals"));
+  const section = guide.slice(guide.indexOf("### Execution and authentication"));
   const bash = section.match(/```sh\n([\s\S]*?)\n```/)[1];
   const powershell = section.match(/```powershell\n([\s\S]*?)\n```/)[1];
   const expectedBash = String.raw`(
@@ -144,8 +152,8 @@ try {
   assert.equal(normalizeSpec(powershell), normalizeSpec(expectedPowerShell));
   const paid = guide.slice(guide.indexOf("## Paid evals"));
   assert.equal(normalizeSpec(paid.match(/```sh\n([\s\S]*?)\n```/)[1]), normalizeSpec(expectedBash),
-    "the first paid run example must include Docker prerequisites and scoped authentication");
-  assert.ok(paid.indexOf("### Native Docker evals") < paid.indexOf("```sh"));
+    "the first paid run example must include scoped authentication");
+  assert.ok(paid.indexOf("### Execution and authentication") < paid.indexOf("```sh"));
   const env = { ...process.env, COPILOT_GITHUB_TOKEN: "synthetic-copilot", GH_TOKEN: "synthetic-gh", GITHUB_TOKEN: "synthetic-github" };
   for (const exit of [0, 37]) {
     const sh = spawnSync("bash", ["-c", `
@@ -210,82 +218,3 @@ exit 0
   assert.ifError(empty.error);
   assert.equal(empty.status, 0, empty.stderr);
 });
-
-test("a missing Docker daemon rejects native preparation rather than selecting host execution", async () => {
-  const backend = new DockerBackend({
-    cwd: root,
-    cli: {
-      contextInspect: async () => ({ name: "test-local", dockerEndpoint: "unix:///var/run/docker.sock" }),
-      info: async () => { throw new Error("test Docker daemon unavailable"); },
-    },
-  });
-  await assert.rejects(backend.prepare({}, { runId: "model-free-docker-unavailable" }),
-    /test Docker daemon unavailable/);
-  await backend.shutdown();
-});
-
-test("native eval and experiment commands fail closed without Docker even with a model-free local executor", (t) => {
-  const directory = resolve(root, ".vally", `docker-unavailable-${randomUUID()}`);
-  mkdirSync(directory, { recursive: true });
-  t.after(() => rmSync(directory, { recursive: true, force: true }));
-  const spec = resolve(directory, "eval.yaml");
-  writeFileSync(spec, [
-    "name: model-free-docker-unavailable",
-    "defaults:",
-    "  executor: mock",
-    "  timeout: 1s",
-    "stimuli:",
-    "  - name: synthetic",
-    "    prompt: model-free",
-    "    graders:",
-    "      - type: completed",
-    "",
-  ].join("\n"));
-  const experiment = resolve(directory, "synthetic.experiment.yaml");
-  writeFileSync(experiment, [
-    "name: model-free-docker-unavailable",
-    "evals: [eval.yaml]",
-    "matrix:",
-    "  timeout:",
-    "    path: /defaults/timeout",
-    "    values: [1s, 2s]",
-    "baseline:",
-    "  timeout: 1s",
-    "execution:",
-    "  workers: 1",
-    "",
-  ].join("\n"));
-  for (const [command, args] of [
-    ["eval", [
-      "eval", "--eval-spec", spec, "--work-dir", root,
-      "--workers", "1", "--runs", "1", "--max-retries", "0", "--shutdown-timeout", "1s",
-    ]],
-    ["experiment", ["experiment", "run", experiment, "--workers", "1"]],
-  ]) {
-    const child = spawnSync(process.execPath, [
-      resolve(root, "node_modules/@microsoft/vally-cli/dist/index.js"),
-      ...args,
-      "--workspace", resolve(directory, `${command}-workspace`),
-      "--output-dir", resolve(directory, `${command}-outputs`),
-      "--require-pass",
-    ], {
-      cwd: root,
-      encoding: "utf8",
-      timeout: 20_000,
-      env: {
-        PATH: directory,
-        ...(process.env.SystemRoot && { SystemRoot: process.env.SystemRoot }),
-        HOME: directory,
-        USERPROFILE: directory,
-        TMP: directory,
-        TEMP: directory,
-        DOCKER_CONFIG: directory,
-        DOCKER_CONTEXT: "model-free-nonexistent-context",
-        VALLY_TELEMETRY_OPTOUT: "1",
-      },
-    });
-    assert.ifError(child.error);
-    assert.notEqual(child.status, 0, child.stdout + child.stderr);
-    assert.match(child.stdout + child.stderr, /docker.*ENOENT|ENOENT.*docker|Docker backend preparation/i);
-  }
-});

.github/workflows/evals.yml

@@ -15,7 +15,7 @@ jobs:
       copilot-requests: write
     env:
       VALLY_TELEMETRY_OPTOUT: '1'
-      # .vally.yaml selects Docker; the appliance supplies the executor runtime.
+      COPILOT_CLI_PATH: ${{ github.workspace }}/node_modules/@github/copilot/npm-loader.js
     steps:
       - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:
@@ -25,7 +25,6 @@ jobs:
           node-version: 24
           cache: npm
       - run: npm ci --no-audit --no-fund
-      - run: test "$(docker info --format '{{.OSType}}')" = linux && docker compose version
       - run: npm run eval -- --suite smoke --require-pass
         env:
           GITHUB_TOKEN: ${{ github.token }}

.vally.yaml

@@ -1,9 +1,4 @@
 # Vally configuration for the included plugins' evals.
-defaults:
-  backend:
-    name: docker
-    args:
-      grade-network: none
 paths:
   # evals/ mirrors plugins/: one eval per skill or agent, such as
   # evals/<plugin>/skills/<skill>/eval.yaml.

README.md

@@ -44,7 +44,6 @@ For marketplace development and evals, use this shared setup.
 - **GitHub Copilot CLI 1.0.85+**.
 - **Vally ([`@microsoft/vally-cli`](https://www.npmjs.com/package/@microsoft/vally-cli)) 0.21.0**, pinned in `package.json`.
 - **Git 2.28.0+**.
-- For paid repository evals, **Docker with Compose and a running Linux daemon**.
 
 ## Make it yours
 
@@ -197,26 +196,22 @@ validation, catalog consistency checks, and remote-pin checks on every pull
 request.
 
 **Paid evals:** first complete the
-[Docker and scoped authentication setup](docs/checks-and-evals.md#native-docker-evals).
+[execution and scoped authentication setup](docs/checks-and-evals.md#execution-and-authentication).
 Local examples use a dedicated Copilot Requests token with public-repository
 access for the synthetic fixtures, not a broad GitHub CLI login credential.
 Then `npm run eval` runs real Copilot sessions and costs AI credits.
 [Evals](.github/workflows/evals.yml) runs only when manually started; pull
 requests never trigger paid evals.
 
-Repository evals **default to Vally's native Docker backend** in
-[.vally.yaml](.vally.yaml), including `npm run eval` and the manually dispatched
-eval workflow. `npm run eval -- --suite full` selects every retained case,
-including the independent reviewer agent. These runs still spend AI credits
-and require Docker, Compose, network access, and container-compatible
-authentication. Missing Docker or image preparation fails the run; it does not
-silently fall back to host execution. Vally builds and caches its version-matched
-default image locally.
-See [Native Docker evals and isolation limits](docs/checks-and-evals.md#native-docker-evals)
-before running. The pinned Evals plugin's Run entrypoint cannot read the new
-project backend default and currently rejects this configuration; use the native
-Vally CLI rather than assuming plugin Run is container-isolated. Paid CI remains
-manual-only.
+Repository evals use **Vally's default local backend**, without an advanced
+project backend setting. This keeps [.vally.yaml](.vally.yaml) within the GitHub
+Copilot Evals tools' supported project configuration. Docker and Compose are not
+required by this default. Execution uses the host, not a container sandbox;
+review eval setup, tools, network access and credential exposure before approval.
+`npm run eval -- --suite full` selects every retained case, including the
+independent reviewer agent, and still spends AI credits. See
+[execution and authentication](docs/checks-and-evals.md#execution-and-authentication)
+for the compatibility and isolation limits. Paid CI remains manual-only.
 
 **Standard models:** `npm run eval:models` uses the native
 [model experiment](evals/standard-models.experiment.yaml) to test every retained