chore(repo-maintain): consolidated repository maintenance and eval v0.24.6 pin update

.github/scripts/lint.test.mjs

@@ -0,0 +1,77 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = fileURLToPath(new URL("../../", import.meta.url));
+
+function fixture(t) {
+  const directory = resolve(root, ".github/scripts", `.lint-test-${randomUUID()}`);
+  mkdirSync(directory);
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  return directory;
+}
+
+test("lint.mjs validates a plugins-only marketplace and check-remote-pins handles catalogs with no remote entries", (t) => {
+  const directory = fixture(t);
+  const pluginDir = resolve(directory, "plugins/sample-tool");
+  const skillDir = resolve(pluginDir, "skills/sample-tool");
+  mkdirSync(skillDir, { recursive: true });
+
+  const manifest = {
+    $schema: "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
+    name: "sample-tool",
+    version: "0.1.0",
+    description: "Summarize synthetic release checklists.",
+  };
+  writeFileSync(resolve(pluginDir, "plugin.json"), JSON.stringify(manifest, null, 2));
+  writeFileSync(
+    resolve(skillDir, "SKILL.md"),
+    [
+      "---",
+      "name: sample-tool",
+      'description: "Summarize synthetic release checklists when asked to format a release checklist."',
+      "---",
+      "",
+      "# Sample tool",
+      "",
+      "Summarize the supplied checklist items concisely.",
+      "",
+    ].join("\n"),
+  );
+  writeFileSync(
+    resolve(directory, "marketplace.json"),
+    JSON.stringify({
+      name: "sample-marketplace",
+      owner: { name: "Synthetic" },
+      plugins: [{
+        name: manifest.name,
+        source: "./plugins/sample-tool",
+        version: manifest.version,
+        description: manifest.description,
+      }],
+    }, null, 2),
+  );
+
+  const lintResult = spawnSync(process.execPath, [resolve(root, ".github/scripts/lint.mjs")], {
+    cwd: directory,
+    encoding: "utf8",
+    timeout: 30_000,
+    env: { ...process.env, VALLY_TELEMETRY_OPTOUT: "1" },
+  });
+  assert.ifError(lintResult.error);
+  assert.equal(lintResult.status, 0, `${lintResult.stdout}${lintResult.stderr}`);
+  assert.match(lintResult.stdout, /sample-tool/);
+
+  const pinsResult = spawnSync(process.execPath, [resolve(root, ".github/scripts/check-remote-pins.mjs")], {
+    cwd: directory,
+    encoding: "utf8",
+    timeout: 10_000,
+  });
+  assert.ifError(pinsResult.error);
+  assert.equal(pinsResult.status, 0, pinsResult.stderr);
+  assert.match(pinsResult.stdout, /No remote pins to check\./);
+});

.github/workflows/validate.yml

@@ -7,9 +7,13 @@ on:
   workflow_dispatch:
 permissions:
   contents: read
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 jobs:
   lint:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
         with:

.vally.yaml

@@ -26,11 +26,11 @@ environments:
     # Vally runs setup in each trial workspace; Copilot discovers these skills natively.
     commands:
       - git clone --filter=blob:none --no-checkout --depth 1 --branch
-        copilot-evals-v0.24.0 --sparse https://github.com/github/evals.git
+        copilot-evals-v0.24.6 --sparse https://github.com/github/evals.git
         .vally-eval-source
       - git -C .vally-eval-source sparse-checkout set packages/plugin/skills
       - git -C .vally-eval-source checkout --detach
-        34ddb5803aeaee8330ab70c19ad1a3f37fb77ded
+        5c3d5213f88149fe1e5476ef029277fe56fd42e4
       - >-
         node -e "const fs = require('node:fs');
         fs.cpSync('.vally-eval-source/packages/plugin/skills', '.github/skills',

README.md

@@ -21,7 +21,7 @@ Included local maintenance plugins and optional remote plugins:
 | [plugin-authoring](plugins/plugin-authoring/) | Scaffold, try locally, and prepare contributions to this plugin marketplace. |
 | [marketplace-maintainer](plugins/marketplace-maintainer/) | Review contributed plugins, maintain remote pins, and prepare marketplace releases. |
 | [pr-description](plugins/pr-description/) | Draft structured pull request descriptions from supplied change notes or diffs without inventing unshipped work. |
-| [GitHub Copilot Evals (Preview)](https://github.com/github/evals/blob/copilot-evals-v0.24.0/README.md) (`eval`, optional remote) | Create, run, review, and compare evals for your Copilot customizations. |
+| [GitHub Copilot Evals (Preview)](https://github.com/github/evals/blob/copilot-evals-v0.24.6/README.md) (`eval`, optional remote) | Create, run, review, and compare evals for your Copilot customizations. |
 
 ## Background
 
@@ -103,7 +103,7 @@ For optional GitHub Copilot Evals, install it explicitly:
 copilot plugin install eval@plugin-marketplace-starter
 ```
 
-See its [upstream setup guidance](https://github.com/github/evals/blob/copilot-evals-v0.24.0/README.md#install),
+See its [upstream setup guidance](https://github.com/github/evals/blob/copilot-evals-v0.24.6/README.md#install),
 including experimental mode for Copilot CLI commands and Canvas. For remote-entry
 checks and the local-directory install limitation, see
 [Remote pin testing](plugins/marketplace-maintainer/skills/update-remote-pins/references/remote-pins.md#testing-a-remote-entry).
@@ -236,7 +236,7 @@ This template and its local plugins use the [MIT License](LICENSE). Keep its
 below it.
 
 Remote plugins retain their upstream licenses. GitHub Copilot Evals is source
-available under the [GitHub Limited Use Source License](https://github.com/github/evals/blob/34ddb5803aeaee8330ab70c19ad1a3f37fb77ded/LICENSE).
+available under the [GitHub Limited Use Source License](https://github.com/github/evals/blob/5c3d5213f88149fe1e5476ef029277fe56fd42e4/LICENSE).
 It isn't MIT-licensed or open source.
 
 ## Maintainers

marketplace.json

@@ -28,10 +28,10 @@
         "source": "github",
         "repo": "github/evals",
         "path": "packages/plugin",
-        "ref": "copilot-evals-v0.24.0",
-        "sha": "34ddb5803aeaee8330ab70c19ad1a3f37fb77ded"
+        "ref": "copilot-evals-v0.24.6",
+        "sha": "5c3d5213f88149fe1e5476ef029277fe56fd42e4"
       },
-      "version": "0.24.0",
+      "version": "0.24.6",
       "description": "GitHub Copilot Evals (Preview): set up a project with /eval setup, create and save evals with /eval create, save revised copies with /eval improve, run with /eval run, review with /eval review, and compare saved runs or explore one skill with /eval compare. Setup is optional; saves use normal host permissions and runs require cost/effect approval.",
       "homepage": "https://github.com/github/evals"
     }