diff --git a/.changes/playgrounds/test-12191-playgrounds-standins-4b7c2e19.md b/.changes/playgrounds/test-12191-playgrounds-standins-4b7c2e19.md new file mode 100644 index 00000000000..f0201a4aa0a --- /dev/null +++ b/.changes/playgrounds/test-12191-playgrounds-standins-4b7c2e19.md @@ -0,0 +1,5 @@ +--- +bump: none +--- + +The playgrounds eval cases list an eval-only `playground` stand-in, so the plugin loads in `claude plugin eval` and its suite measures something. diff --git a/plugins/playgrounds/evals/use-absent-install-scope/case.yaml b/plugins/playgrounds/evals/use-absent-install-scope/case.yaml index 77a7af9846c..5e2ddf7083d 100644 --- a/plugins/playgrounds/evals/use-absent-install-scope/case.yaml +++ b/plugins/playgrounds/evals/use-absent-install-scope/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-absent-install-scope +plugins: ["../..", "playground-stub"] graders: - name: install-command type: regex diff --git a/plugins/playgrounds/evals/use-absent-install-scope/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-absent-install-scope/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-absent-install-scope/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-boundary-visual-form/case.yaml b/plugins/playgrounds/evals/use-boundary-visual-form/case.yaml index 17b0057c478..49720e159d9 100644 --- a/plugins/playgrounds/evals/use-boundary-visual-form/case.yaml +++ b/plugins/playgrounds/evals/use-boundary-visual-form/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-boundary-visual-form +plugins: ["../..", "playground-stub"] graders: - name: no-playground-skill type: tool_used diff --git a/plugins/playgrounds/evals/use-boundary-visual-form/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-boundary-visual-form/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-boundary-visual-form/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-install-policy-blocked/case.yaml b/plugins/playgrounds/evals/use-install-policy-blocked/case.yaml index 291ab60badf..409d6cc2eca 100644 --- a/plugins/playgrounds/evals/use-install-policy-blocked/case.yaml +++ b/plugins/playgrounds/evals/use-install-policy-blocked/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-install-policy-blocked +plugins: ["../..", "playground-stub"] graders: - name: names-policy-block type: regex diff --git a/plugins/playgrounds/evals/use-install-policy-blocked/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-install-policy-blocked/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-install-policy-blocked/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-reads-consumer-notes/case.yaml b/plugins/playgrounds/evals/use-reads-consumer-notes/case.yaml index e5df5b17da0..f098a644ce8 100644 --- a/plugins/playgrounds/evals/use-reads-consumer-notes/case.yaml +++ b/plugins/playgrounds/evals/use-reads-consumer-notes/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-reads-consumer-notes +plugins: ["../..", "playground-stub"] graders: - name: reads-consumer-notes type: tool_used diff --git a/plugins/playgrounds/evals/use-reads-consumer-notes/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-reads-consumer-notes/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-reads-consumer-notes/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-routes-natural-explorer/case.yaml b/plugins/playgrounds/evals/use-routes-natural-explorer/case.yaml index 31d7ea314ca..756129880fe 100644 --- a/plugins/playgrounds/evals/use-routes-natural-explorer/case.yaml +++ b/plugins/playgrounds/evals/use-routes-natural-explorer/case.yaml @@ -1,5 +1,6 @@ schema_version: "1.1" name: use-routes-natural-explorer +plugins: ["../..", "playground-stub"] graders: - name: routes-use type: tool_used diff --git a/plugins/playgrounds/evals/use-routes-natural-explorer/playground-stub/.claude-plugin/plugin.json b/plugins/playgrounds/evals/use-routes-natural-explorer/playground-stub/.claude-plugin/plugin.json new file mode 100644 index 00000000000..6e67d5f1b7d --- /dev/null +++ b/plugins/playgrounds/evals/use-routes-natural-explorer/playground-stub/.claude-plugin/plugin.json @@ -0,0 +1,5 @@ +{ + "name": "playground", + "description": "Eval-only stand-in for playground@claude-plugins-official with no skill, so the playgrounds plugin loads while its upstream skill stays absent.", + "version": "0.0.0" +} diff --git a/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md b/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md index c3ebb410f09..f7c731d1610 100644 --- a/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md +++ b/plugins/playgrounds/evals/use-routes-natural-explorer/prompt.md @@ -1,5 +1,5 @@ --- -description: "Routing from plain words (SKILL.md line 2: 'interactive explorer for X', 'tune a parameter space visually') in an eval run, where the upstream playground plugin is never installed (the run copy drops the manifest dependencies key). The skill must reach Step 3 (lines 65-75) and print the marketplace-qualified install command, and must not build the explorer itself (lines 18-19 and 147-148: this skill never builds a playground). Write is granted so the no-write grader can fail: a baseline is expected to hand-build the HTML page. The marketplace-add line is not graded, since lines 72-74 and 78-79 disagree on when it belongs. Needs --allow-tools Write." +description: "Routing from plain words (SKILL.md line 2: 'interactive explorer for X', 'tune a parameter space visually') in an eval run, where the upstream playground plugin is never installed (a stand-in with no skill meets the manifest dependency). This is an absent-path case: allowed_tools has no Bash, so Step 1 cannot run `claude plugin list` and falls back to the skill listing (lines 28-32), where the stand-in lists no `playground` skill. The skill must reach Step 3 (lines 65-75) and print the marketplace-qualified install command, and must not build the explorer itself (lines 18-19 and 147-148: this skill never builds a playground). Write is granted so the no-write grader can fail: a baseline is expected to hand-build the HTML page. The marketplace-add line is not graded, since lines 72-74 and 78-79 disagree on when it belongs. Needs --allow-tools Write." tags: [playgrounds, use, outcome, routing] max_turns: 15 timeout_seconds: 600