From f30b68da451518d6644f08e4783dbcf7f7f62d01 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 2 Nov 2026 22:30:43 -0500 Subject: [PATCH 1/2] test(playgrounds): load the plugin in its eval suite through a stand-in The manifest's cross-marketplace dependency kept the plugin from loading in `claude plugin eval`, so both arms ran without it. Each case now lists an eval-only `playground` stand-in with no skill, in its own subdirectory because the CLI refuses a plugins entry outside the case. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VQb9Q296cDKeBR2kaZuHaV --- .../playgrounds/test-12191-playgrounds-standins-4b7c2e19.md | 5 +++++ plugins/playgrounds/evals/use-absent-install-scope/case.yaml | 1 + .../playground-stub/.claude-plugin/plugin.json | 5 +++++ plugins/playgrounds/evals/use-boundary-visual-form/case.yaml | 1 + .../playground-stub/.claude-plugin/plugin.json | 5 +++++ .../playgrounds/evals/use-install-policy-blocked/case.yaml | 1 + .../playground-stub/.claude-plugin/plugin.json | 5 +++++ plugins/playgrounds/evals/use-reads-consumer-notes/case.yaml | 1 + .../playground-stub/.claude-plugin/plugin.json | 5 +++++ .../playgrounds/evals/use-routes-natural-explorer/case.yaml | 1 + .../playground-stub/.claude-plugin/plugin.json | 5 +++++ .../playgrounds/evals/use-routes-natural-explorer/prompt.md | 2 +- 12 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 .changes/playgrounds/test-12191-playgrounds-standins-4b7c2e19.md create mode 100644 plugins/playgrounds/evals/use-absent-install-scope/playground-stub/.claude-plugin/plugin.json create mode 100644 plugins/playgrounds/evals/use-boundary-visual-form/playground-stub/.claude-plugin/plugin.json create mode 100644 plugins/playgrounds/evals/use-install-policy-blocked/playground-stub/.claude-plugin/plugin.json create mode 100644 plugins/playgrounds/evals/use-reads-consumer-notes/playground-stub/.claude-plugin/plugin.json create mode 100644 plugins/playgrounds/evals/use-routes-natural-explorer/playground-stub/.claude-plugin/plugin.json diff --git a/.changes/playgrounds/test-12191-playgrounds-standins-4b7c2e19.md b/.changes/playgrounds/test-12191-playgrounds-standins-4b7c2e19.md new file mode 100644 index 00000000000..f0201a4aa0a --- /dev/null +++ b/.changes/playgrounds/test-12191-playgrounds-standins-4b7c2e19.md @@ -0,0 +1,5 @@ +--- +bump: none +--- + +The playgrounds eval cases list an eval-only `playground` stand-in, so the plugin loads in `claude plugin eval` and its suite measures something. diff --git a/plugins/playgrounds/evals/use-absent-install-scope/case.yaml b/plugins/playgrounds/evals/use-absent-install-scope/case.yaml index 77a7af9846c..5e2ddf7083d 100644 --- a/plugins/playgrounds/evals/use-absent-install-scope/case.yaml +++ b/plugins/playgrounds/evals/use-absent-install-scope/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-absent-install-scope +plugins: ["../..", "playground-stub"] graders: - name: install-command type: regex diff --git a/plugins/playgrounds/evals/use-absent-install-scope/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-absent-install-scope/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-absent-install-scope/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-boundary-visual-form/case.yaml b/plugins/playgrounds/evals/use-boundary-visual-form/case.yaml index 17b0057c478..49720e159d9 100644 --- a/plugins/playgrounds/evals/use-boundary-visual-form/case.yaml +++ b/plugins/playgrounds/evals/use-boundary-visual-form/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-boundary-visual-form +plugins: ["../..", "playground-stub"] graders: - name: no-playground-skill type: tool_used diff --git a/plugins/playgrounds/evals/use-boundary-visual-form/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-boundary-visual-form/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-boundary-visual-form/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-install-policy-blocked/case.yaml b/plugins/playgrounds/evals/use-install-policy-blocked/case.yaml index 291ab60badf..409d6cc2eca 100644 --- a/plugins/playgrounds/evals/use-install-policy-blocked/case.yaml +++ b/plugins/playgrounds/evals/use-install-policy-blocked/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-install-policy-blocked +plugins: ["../..", "playground-stub"] graders: - name: names-policy-block type: regex diff --git a/plugins/playgrounds/evals/use-install-policy-blocked/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-install-policy-blocked/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-install-policy-blocked/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-reads-consumer-notes/case.yaml b/plugins/playgrounds/evals/use-reads-consumer-notes/case.yaml index e5df5b17da0..f098a644ce8 100644 --- a/plugins/playgrounds/evals/use-reads-consumer-notes/case.yaml +++ b/plugins/playgrounds/evals/use-reads-consumer-notes/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-reads-consumer-notes +plugins: ["../..", "playground-stub"] graders: - name: reads-consumer-notes type: tool_used diff --git a/plugins/playgrounds/evals/use-reads-consumer-notes/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-reads-consumer-notes/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-reads-consumer-notes/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-routes-natural-explorer/case.yaml b/plugins/playgrounds/evals/use-routes-natural-explorer/case.yaml index 31d7ea314ca..756129880fe 100644 --- a/plugins/playgrounds/evals/use-routes-natural-explorer/case.yaml +++ b/plugins/playgrounds/evals/use-routes-natural-explorer/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-routes-natural-explorer +plugins: ["../..", "playground-stub"] graders: - name: routes-use type: tool_used diff --git a/plugins/playgrounds/evals/use-routes-natural-explorer/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-routes-natural-explorer/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-routes-natural-explorer/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md b/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md index c3ebb410f09..70367924459 100644 --- a/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md +++ b/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md @@ -1,5 +1,5 @@ --- -description: "Routing from plain words (SKILL.md line 2: 'interactive explorer for X', 'tune a parameter space visually') in an eval run, where the upstream playground plugin is never installed (the run copy drops the manifest dependencies key). The skill must reach Step 3 (lines 65-75) and print the marketplace-qualified install command, and must not build the explorer itself (lines 18-19 and 147-148: this skill never builds a playground). Write is granted so the no-write grader can fail: a baseline is expected to hand-build the HTML page. The marketplace-add line is not graded, since lines 72-74 and 78-79 disagree on when it belongs. Needs --allow-tools Write." +description: "Routing from plain words (SKILL.md line 2: 'interactive explorer for X', 'tune a parameter space visually') in an eval run, where the upstream playground plugin is never installed (a stand-in with no skill meets the manifest dependency). The skill must reach Step 3 (lines 65-75) and print the marketplace-qualified install command, and must not build the explorer itself (lines 18-19 and 147-148: this skill never builds a playground). Write is granted so the no-write grader can fail: a baseline is expected to hand-build the HTML page. The marketplace-add line is not graded, since lines 72-74 and 78-79 disagree on when it belongs. Needs --allow-tools Write." tags: [playgrounds, use, outcome, routing] max_turns: 15 timeout_seconds: 600 From e8b9a63ac288a081f37f032e39792076cc75ec36 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 3 Nov 2026 05:50:38 -0500 Subject: [PATCH 2/2] test(playgrounds): say why the routing case reads the stand-in as absent The routing case's allowed_tools has no Bash, so the use skill's presence check cannot run `claude plugin list` and falls back to the skill listing, where the skill-less playground stand-in lists nothing. The description now states that chain, which is why the case stays absent-path. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Vcupom3BKF6QuC5U3sRJPj --- plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md b/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md index 70367924459..f7c731d1610 100644 --- a/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md +++ b/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md @@ -1,5 +1,5 @@ --- -description: "Routing from plain words (SKILL.md line 2: 'interactive explorer for X', 'tune a parameter space visually') in an eval run, where the upstream playground plugin is never installed (a stand-in with no skill meets the manifest dependency). The skill must reach Step 3 (lines 65-75) and print the marketplace-qualified install command, and must not build the explorer itself (lines 18-19 and 147-148: this skill never builds a playground). Write is granted so the no-write grader can fail: a baseline is expected to hand-build the HTML page. The marketplace-add line is not graded, since lines 72-74 and 78-79 disagree on when it belongs. Needs --allow-tools Write." +description: "Routing from plain words (SKILL.md line 2: 'interactive explorer for X', 'tune a parameter space visually') in an eval run, where the upstream playground plugin is never installed (a stand-in with no skill meets the manifest dependency). This is an absent-path case: allowed_tools has no Bash, so Step 1 cannot run `claude plugin list` and falls back to the skill listing (lines 28-32), where the stand-in lists no `playground` skill. The skill must reach Step 3 (lines 65-75) and print the marketplace-qualified install command, and must not build the explorer itself (lines 18-19 and 147-148: this skill never builds a playground). Write is granted so the no-write grader can fail: a baseline is expected to hand-build the HTML page. The marketplace-add line is not graded, since lines 72-74 and 78-79 disagree on when it belongs. Needs --allow-tools Write." tags: [playgrounds, use, outcome, routing] max_turns: 15 timeout_seconds: 600