Compare changed skills through the reusable Evals Action

.github/scripts/pr-description.test.mjs

@@ -0,0 +1,130 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs";
+import { resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { createDefaultGraderRegistry } from "@microsoft/vally";
+import { parse } from "yaml";
+
+const root = fileURLToPath(new URL("../../", import.meta.url));
+const prDescriptionTest = existsSync(resolve(root, "plugins/pr-description")) ? test : test.skip;
+
+prDescriptionTest("pr-description eval stages catalog-local skills and enforces non-collision", async () => {
+  const evalDir = resolve(root, "evals/pr-description/skills/pr-description");
+  const spec = parse(readFileSync(resolve(evalDir, "eval.yaml"), "utf8"));
+  assert.equal(spec.scoring.threshold, 1);
+  const project = parse(readFileSync(resolve(root, ".vally.yaml"), "utf8"));
+  assert.equal(spec.agent_environment, "marketplace");
+  const skills = project.environments.marketplace.skills;
+  for (const configuredSkill of skills) {
+    assert.ok(existsSync(resolve(evalDir, configuredSkill)), `configured cohort skill path is missing: ${configuredSkill}`);
+  }
+  const staged = new Set(skills.map((skill) => resolve(evalDir, skill)));
+  const catalog = JSON.parse(readFileSync(resolve(root, "marketplace.json"), "utf8"));
+  for (const plugin of catalog.plugins.filter((plugin) => typeof plugin.source === "string")) {
+    const directory = resolve(root, plugin.source, "skills");
+    if (!existsSync(directory)) continue;
+    for (const skill of readdirSync(directory)) {
+      const source = resolve(directory, skill);
+      if (existsSync(resolve(source, "SKILL.md"))) {
+        assert.ok(staged.has(source), `Add catalog skill ${plugin.name}/${skill} to the native marketplace environment.`);
+      }
+    }
+  }
+
+  const grader = createDefaultGraderRegistry().get("skill-invocation");
+  const positive = spec.stimuli.find((item) => item.tags?.control === "positive");
+  const negative = spec.stimuli.find((item) => item.tags?.control === "negative");
+  const posConfig = positive.graders.find((item) => item.type === "skill-invocation").config;
+  const negConfig = negative.graders.find((item) => item.type === "skill-invocation").config;
+  const activation = (name) => ({ type: "skill_activation", data: { name } });
+
+  const exactMatch = await grader.grade({
+    stimulus: positive, config: posConfig,
+    trajectory: { output: "ok", events: [activation("pr-description")] },
+  });
+  assert.equal(exactMatch.passed, true);
+
+  if (posConfig.disallowed?.length) {
+    const neighborCollision = await grader.grade({
+      stimulus: positive, config: posConfig,
+      trajectory: { output: "ok", events: [activation("pr-description"), activation(posConfig.disallowed[0])] },
+    });
+    assert.equal(neighborCollision.passed, false, "activating a configured neighbor must fail");
+  }
+
+  const negativeTriggered = await grader.grade({
+    stimulus: negative, config: negConfig,
+    trajectory: { output: "ok", events: [activation("pr-description")] },
+  });
+  assert.equal(negativeTriggered.passed, false, "activating pr-description on the negative case must fail");
+});
+
+prDescriptionTest("pr-description program grader enforces ordered headings, factual changes, and unshipped backlog exclusion", async (t) => {
+  const gradingDir = resolve(root, ".github/scripts", `.pr-grader-test-${randomUUID()}`);
+  mkdirSync(gradingDir, { recursive: true });
+  t.after(() => rmSync(gradingDir, { recursive: true, force: true }));
+  copyFileSync(resolve(root, "evals/pr-description/skills/pr-description/grader.mjs"), resolve(gradingDir, "grader.mjs"));
+  copyFileSync(resolve(root, "evals/_shared/graders/common.mjs"), resolve(gradingDir, "common.mjs"));
+  const { evaluate: evaluatePrDescription } = await import(pathToFileURL(resolve(gradingDir, "grader.mjs")));
+  const { score } = await import(pathToFileURL(resolve(gradingDir, "common.mjs")));
+  const validOutput = [
+    "## Summary",
+    "Validate `--timeout` as a positive integer in `src/cli.mjs` and add unit tests. Retry backoff (#42) remains open and out of scope.",
+    "",
+    "## Changes",
+    "- Added positive integer validation for `--timeout` in `src/cli.mjs`.",
+    "- Added unit tests for `--timeout 0` and `--timeout -5` in `test/cli.test.mjs`.",
+    "",
+    "## Verification",
+    "- `npm test` passed (14 tests).",
+    "",
+    "## Risks / Rollback",
+    "- Low risk; revert the commit if needed.",
+  ].join("\n");
+  assert.equal(score("pr-description-contract", evaluatePrDescription({ output: validOutput, events: [] })).passed, true);
+  for (const passingLine of [
+    "- `npm test` (14 tests): passed.",
+    "- `npm test` \u2014 14 tests, all passing.",
+    "- Ran `npm test`; 14/14 tests passed.",
+    "- `npm test`: 14 tests passed, 0 failed, 0 errors.",
+    "- `npm test` passed (14 tests, no errors).",
+    "- `npm test`: 14 passed, # fail 0, 0 failures.",
+  ]) {
+    const output = validOutput.replace("- `npm test` passed (14 tests).", passingLine);
+    assert.equal(score("pr-description-contract", evaluatePrDescription({ output, events: [] })).passed, true, passingLine);
+  }
+  for (const negatedLine of [
+    "- `npm test` (14 tests).",
+    "- `npm test` did not pass (14 tests).",
+    "- `npm test` (14 tests): not passing.",
+    "- `npm test`: 14 tests; 12 passed, 2 failures.",
+    "- `npm test` failed (14 tests).",
+  ]) {
+    const output = validOutput.replace("- `npm test` passed (14 tests).", negatedLine);
+    const result = score("pr-description-contract", evaluatePrDescription({ output, events: [] }));
+    assert.equal(result.passed, false, negatedLine);
+    assert.match(result.evidence, /Recorded the supplied npm test verification result/);
+  }
+  for (const leakedChanges of [
+    "- Added unit tests for `--timeout 0` and `--timeout -5` in `test/cli.test.mjs`.\n- Implemented exponential retry backoff (#42).",
+    "- Rejects `--timeout` values that are not positive integers.\n- Adds exponential retry backoff (#42).",
+  ]) {
+    const output = validOutput.replace(
+      "- Added unit tests for `--timeout 0` and `--timeout -5` in `test/cli.test.mjs`.", leakedChanges);
+    const result = score("pr-description-contract", evaluatePrDescription({ output, events: [] }));
+    assert.equal(result.passed, false);
+    assert.match(result.evidence, /Excluded unshipped retry backoff/);
+  }
+  const missingRisks = validOutput.split("## Risks / Rollback")[0];
+  assert.equal(score("pr-description-contract", evaluatePrDescription({ output: missingRisks, events: [] })).passed, false);
+  for (const prohibitedTool of ["bash", "functions.bash", "functions.powershell", "mcp:edit", "tools/write_file"]) {
+    const result = score("pr-description-contract", evaluatePrDescription({
+      output: validOutput,
+      events: [{ type: "tool_call", data: { toolCallId: "1", toolName: prohibitedTool, arguments: { command: "git status" } } }],
+    }));
+    assert.equal(result.passed, false, `must reject prohibited tool ${prohibitedTool}`);
+    assert.match(result.evidence, /Did not run shell commands or create a pull request/);
+  }
+});

.github/scripts/select-skill-evals.mjs

@@ -0,0 +1,108 @@
+import { execFileSync, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { appendFileSync, existsSync, readFileSync, readdirSync, realpathSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { githubApi } from "./check-remote-pins.mjs";
+
+const name = /^[a-z0-9][a-z0-9._-]*$/;
+const sha = /^[a-f0-9]{40}$/;
+const parse = (contents) => {
+  try { return JSON.parse(contents); } catch { throw new Error("Invalid skill selection JSON."); }
+};
+const json = (path) => parse(readFileSync(path, "utf8"));
+
+function pinnedTree(repo, commit) {
+  const response = githubApi(`repos/${repo}/git/trees/${commit}?recursive=1`, (command, args, options) =>
+    spawnSync(command, args, { ...options, timeout: 30000, maxBuffer: 8 * 1024 * 1024 }));
+  if (response.status !== 200) throw new Error(`Pinned skill inventory returned GitHub HTTP ${response.status}.`);
+  return parse(response.body);
+}
+
+export function inventorySkills(root, readTree = pinnedTree) {
+  const skills = [];
+  for (const plugin of json(resolve(root, "marketplace.json")).plugins) {
+    if (!name.test(plugin.name)) throw new Error("Unsupported catalog plugin name.");
+    let names;
+    if (typeof plugin.source === "string") {
+      if (plugin.source !== `./plugins/${plugin.name}`) throw new Error("Expected a self-contained local plugin path.");
+      const directory = resolve(root, plugin.source, "skills");
+      names = existsSync(directory) ? readdirSync(directory).filter((skill) =>
+        name.test(skill) && existsSync(resolve(directory, skill, "SKILL.md"))) : [];
+    } else {
+      const source = plugin.source;
+      if (source?.source !== "github" || !/^[\w.-]+\/[\w.-]+$/.test(source.repo) || !sha.test(source.sha)) {
+        throw new Error("Remote skill inventory requires a pinned GitHub source.");
+      }
+      const prefix = source.path ? `${source.path}/` : "";
+      if (prefix.split("/").includes("..")) throw new Error("Unsupported remote plugin path.");
+      const tree = readTree(source.repo, source.sha);
+      if (tree.truncated || !Array.isArray(tree.tree)) throw new Error("Incomplete pinned remote skill inventory.");
+      names = tree.tree.flatMap((entry) => {
+        if (entry.type !== "blob" || !entry.path.startsWith(`${prefix}skills/`)) return [];
+        const parts = entry.path.slice(`${prefix}skills/`.length).split("/");
+        return parts.length === 2 && parts[1] === "SKILL.md" && name.test(parts[0]) ? [parts[0]] : [];
+      });
+    }
+    for (const skill of names) skills.push({
+      key: `${plugin.name}/${skill}`,
+      hasEval: existsSync(resolve(root, "evals", plugin.name, "skills", skill, "eval.yaml")),
+    });
+  }
+  return skills.sort((a, b) => a.key.localeCompare(b.key));
+}
+
+export function selectSkillEvals(root, context, environment, readTree = pinnedTree) {
+  if (!environment.protection_rules?.some((rule) => rule.type === "required_reviewers" && rule.reviewers?.length)) {
+    throw new Error("Configure the evals environment with required maintainer reviewers before enabling paid PR evals.");
+  }
+  const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
+  if (![context.base, context.head, context.merge].every((value) => sha.test(value ?? "")) ||
+      git("rev-parse", "HEAD") !== context.merge || git("rev-parse", "HEAD^1") !== context.base ||
+      git("rev-parse", "HEAD^2") !== context.head) {
+    throw new Error("Expected the verified PR merge and its exact parents.");
+  }
+  const changed = execFileSync("git", ["diff", "--no-renames", "--name-only", "-z", context.base, "HEAD", "--", "plugins"],
+    { cwd: root, encoding: "utf8" });
+  const keys = [...new Set(changed.split("\0").flatMap((file) => {
+    const match = /^plugins\/([^/]+)\/skills\/([^/]+)\//.exec(file);
+    return match ? [`${match[1]}/${match[2]}`] : [];
+  }))].sort();
+  const include = [];
+  for (const key of keys) {
+    if (key.split("/").length !== 2 || !key.split("/").every((part) => name.test(part))) {
+      throw new Error("Unsupported plugin or skill path in PR changes.");
+    }
+    const [plugin, skill] = key.split("/");
+    const directory = `plugins/${plugin}/skills/${skill}`;
+    const evalPath = `evals/${plugin}/skills/${skill}/eval.yaml`;
+    const present = (ref) => {
+      const result = spawnSync("git", ["cat-file", "-e", `${ref}:${directory}/SKILL.md`], { cwd: root, stdio: "ignore" });
+      if (result.error) throw result.error;
+      if (result.status !== 0 && result.status !== 128) throw new Error("Could not inspect the skill source tree.");
+      return result.status === 0;
+    };
+    if (present(context.base) && present(context.head) && existsSync(resolve(root, evalPath))) {
+      include.push({ key, id: `skill-${createHash("sha256").update(key).digest("hex").slice(0, 12)}`,
+        directory, evalPath, base: context.base, head: context.head });
+    }
+  }
+  return { ...context, include, changed: keys, skills: inventorySkills(root, readTree) };
+}
+
+if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))) {
+  try {
+    const [contextFile, environmentFile, outputFile] = process.argv.slice(2);
+    if (!contextFile || !environmentFile || !outputFile) throw new Error("Expected context, environment and selection file paths.");
+    const selection = selectSkillEvals(process.cwd(), json(contextFile), json(environmentFile));
+    for (const key of selection.changed.filter((key) => !selection.include.some((entry) => entry.key === key))) {
+      console.log(`::warning::No before/after pair for ${key}: both skill versions and its repo eval are required. No eval was run.`);
+    }
+    writeFileSync(outputFile, `${JSON.stringify(selection)}\n`);
+    appendFileSync(process.env.GITHUB_OUTPUT,
+      `matrix=${JSON.stringify({ include: selection.include })}\ncount=${selection.include.length}\nattempt=${selection.attempt}\nselection=${JSON.stringify(selection)}\n`);
+  } catch (error) {
+    console.error(error.message);
+    process.exitCode = 1;
+  }
+}