Validate full plugin v1.1.0 demo with Scanner v3.4.1

.github/workflows/scan-demo-site.yml

@@ -0,0 +1,108 @@
+name: Scan controlled demo site
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_live_quality:
+        description: Run the optional credentialed GitHub Models quality demo
+        required: false
+        default: false
+        type: boolean
+  pull_request:
+    paths:
+      - .github/workflows/scan-demo-site.yml
+      - example/site-with-errors/**
+
+permissions:
+  contents: read
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+          bundler-cache: true
+          working-directory: example/site-with-errors
+
+      - name: Setup Node
+        uses: actions/setup-node@v7
+        with:
+          node-version: 24
+          cache: npm
+
+      - name: Install plugin dependencies
+        run: npm ci
+
+      - name: Install Playwright browser
+        run: npx playwright install --with-deps chromium
+
+      - name: Build demo site
+        working-directory: example/site-with-errors
+        run: bundle exec jekyll build
+
+      - name: Serve demo site
+        run: |
+          python3 -m http.server 4000 --directory example/site-with-errors/_site > "$RUNNER_TEMP/demo-site.log" 2>&1 &
+          echo "$!" > "$RUNNER_TEMP/demo-site.pid"
+
+          for attempt in {1..10}; do
+            if curl --fail --silent --show-error http://127.0.0.1:4000/alt-text-errors/ > /dev/null; then
+              exit 0
+            fi
+            sleep 1
+          done
+
+          cat "$RUNNER_TEMP/demo-site.log"
+          exit 1
+
+      - name: Produce deterministic demo evidence
+        run: npm run --silent demo:verify > "$RUNNER_TEMP/demo-evidence.json"
+
+      - name: Run optional live quality demo
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_live_quality }}
+        env:
+          GITHUB_MODELS_TOKEN: ${{ secrets.GH_MODELS_TOKEN }}
+          AZURE_VISION_ENDPOINT: ${{ secrets.AZURE_VISION_ENDPOINT }}
+          AZURE_VISION_KEY: ${{ secrets.AZURE_VISION_KEY }}
+        run: npm run --silent demo:live > "$RUNNER_TEMP/live-demo-evidence.json"
+
+      - name: Run accessibility scanner
+        id: scanner
+        uses: github/accessibility-scanner@745198705c4aecb2af83732eaafe1f6e3f7787c2 # v3.4.1
+        with:
+          urls: http://127.0.0.1:4000/alt-text-errors/
+          cache_key: cached_findings-controlled-demo-${{ github.run_id }}.json
+          repository: ${{ github.repository }}
+          token: ${{ github.token }}
+          skip_copilot_assignment: true
+          dry_run: true
+          scans: |
+            ["axe", {"name": "alt-text-scan", "package": "@github/accessibility-scanner-alt-text-plugin", "version": "1.1.0"}]
+
+      - name: Stage demo evidence
+        if: ${{ always() && steps.scanner.outputs.results_file }}
+        run: |
+          mkdir -p "$RUNNER_TEMP/controlled-demo-evidence"
+          cp "${{ steps.scanner.outputs.results_file }}" "$RUNNER_TEMP/controlled-demo-evidence/scanner-results.json"
+          cp "$RUNNER_TEMP/demo-evidence.json" "$RUNNER_TEMP/controlled-demo-evidence/demo-evidence.json"
+          if [[ -f "$RUNNER_TEMP/live-demo-evidence.json" ]]; then
+            cp "$RUNNER_TEMP/live-demo-evidence.json" "$RUNNER_TEMP/controlled-demo-evidence/live-demo-evidence.json"
+          fi
+
+      - name: Upload scanner results
+        if: ${{ always() && steps.scanner.outputs.results_file }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: controlled-demo-results-${{ github.run_id }}
+          path: ${{ runner.temp }}/controlled-demo-evidence
+          if-no-files-found: error
+
+      - name: Stop demo site
+        if: ${{ always() }}
+        run: kill "$(cat "$RUNNER_TEMP/demo-site.pid")"

example/site-with-errors/README.md

@@ -14,10 +14,10 @@ Use it for:
   suite (see [`tests/example-site.test.ts`](../../tests/example-site.test.ts)),
   so the rules stay exercised against real markup in CI.
 
-## Image → rule mapping
+## Deterministic image → rule mapping
 
-Every image on [`alt-text-errors.html`](alt-text-errors.html) points at the same
-placeholder SVG (`assets/img/test-image.svg`); only the `alt` attribute differs.
+The first section of [`alt-text-errors.html`](alt-text-errors.html) exercises
+each rule that runs by default in plugin v1.1.0.
 
 | Image `alt` value            | Rule triggered         | Why it triggers                                  |
 | ---------------------------- | ---------------------- | ------------------------------------------------ |
@@ -27,6 +27,57 @@ placeholder SVG (`assets/img/test-image.svg`); only the `alt` attribute differs.
 | `image`                      | `vague-alt-text`       | A single generic word that describes nothing.    |
 | `company logo` (×2 in a row) | `repeated-alt-text`    | Two consecutive images share identical alt text. |
 
+These are real, credential-free plugin findings. The dedicated
+[`scan-demo-site.yml`](../../.github/workflows/scan-demo-site.yml) workflow
+builds this Jekyll site, serves only `/alt-text-errors/`, and runs Scanner
+v3.4.1 with Axe plus the npm-published plugin v1.1.0. It uses scanner dry-run
+mode and uploads `scanner-results.json`, so the workflow proves npm
+installation and execution without writing issues.
+
+## Model-backed quality cases
+
+The second section contains four inputs for the opt-in `alt-text-quality` rule:
+
+| Case             | Expected mocked verdict | Evidence shown                       |
+| ---------------- | ----------------------- | ------------------------------------ |
+| Keyword stuffing | `needs-fix`             | Tailored SEO-abuse finding           |
+| Inaccurate alt   | `needs-fix`             | Finding with a suggested replacement |
+| Decorative image | `decorative`            | Recommendation to use `alt=""`       |
+| Accurate control | `ok`                    | No finding                           |
+
+These outcomes are **mocked test evidence**, not live model results. The
+targeted test injects fixed judge verdicts, then runs the production
+`alt-text-quality` rule-to-finding mapping. This keeps the demo deterministic
+and credential-free while clearly showing behavior that would otherwise
+require GitHub Models and, optionally, Azure AI Vision.
+
+The credential-free verifier prints all demo evidence as JSON:
+
+```sh
+npm run demo:verify
+```
+
+Its output separates:
+
+- the real deterministic plugin findings,
+- mocked quality verdicts using real context extraction, remediation mapping,
+  and scanner finding emission, and
+- mocked Azure caption, OCR, and tag signals passed through the production
+  Azure context-enrichment layer.
+
+For an optional live run, set `GITHUB_MODELS_TOKEN` to a PAT with `models:read`
+and run:
+
+```sh
+npm run demo:live
+```
+
+If `AZURE_VISION_ENDPOINT` and `AZURE_VISION_KEY` are also set, the live command
+automatically requests Azure-augmented mode. Set
+`ALT_TEXT_JUDGE_MODE=copilot` or `ALT_TEXT_JUDGE_MODE=azure-augmented` to force
+a mode. Live output is credentialed and nondeterministic; it is not part of the
+required CI evidence.
+
 ## Run it locally
 
 The site is a standard Jekyll site served as a static build behind Rack/Puma.
@@ -54,12 +105,42 @@ You don't need Ruby or a running server to confirm the plugin flags this page.
 From the repository root:
 
 ```sh
-npm install
+npm ci
 npx playwright install chromium
-npm test
+npm test -- tests/example-site.test.ts tests/unit/azure-augmented-judge.test.ts --reporter=verbose
 ```
 
-The `example site-with-errors` test loads
+The targeted tests load
 [`alt-text-errors.html`](alt-text-errors.html), runs the real `alt-text-scan`
-plugin against it, and asserts that every rule in the table above produces a
-finding.
+plugin against it, asserts exactly one finding for each deterministic rule,
+checks the four model cases through fixed fake-judge verdicts, and exercise the
+production Azure enrichment and fallback layers with a fake Azure client. No
+model or Azure credentials are used.
+
+## Evidence and limitations
+
+| Feature                          | Evidence layer                 | Expected result                                                             |
+| -------------------------------- | ------------------------------ | --------------------------------------------------------------------------- |
+| Scanner v3.4.1 action            | Hosted workflow                | Immutable scanner commit is downloaded and the scan step succeeds.          |
+| npm plugin loading               | Hosted workflow                | npm installs v1.1.0, then Scanner discovers and runs `alt-text-scan`.       |
+| Five deterministic rules         | Hosted artifact and local test | Exactly one plugin finding for each rule.                                   |
+| Rule configuration               | Local test                     | Disabling `missing-alt-text` suppresses only that finding.                  |
+| Context extraction               | Local test                     | Model input includes the page title, nearest heading, and figure caption.   |
+| Keyword stuffing                 | Fixed fake judge               | Production mapping emits the tailored SEO-abuse finding.                    |
+| Inaccurate alt and remediation   | Fixed fake judge               | Production mapping emits an `inaccurate` finding and suggested replacement. |
+| Decorative and accurate controls | Fixed fake judge               | Decorative yields an `alt=""` recommendation; accurate yields no finding.   |
+| Azure caption, OCR, and tags     | Fake Azure client              | Production enrichment adds high-confidence signals to model context.        |
+| Azure failure fallback           | Fake Azure client              | Production enrichment falls back to unmodified Copilot-only context.        |
+| Axe alongside the plugin         | Hosted artifact                | Axe findings and plugin findings appear in the same scanner results file.   |
+
+The hosted Scanner action proves only behavior available through the published
+npm package without model credentials: npm loading, Axe, and the five default
+rules. Scanner cannot inject the repository's fake judge into the published
+package, so model and Azure evidence comes from the checked-in v1.1.0 source
+through `demo:verify` and targeted tests. The optional `demo:live` path is the
+only evidence that calls GitHub Models or Azure; its output is intentionally
+not asserted in CI because it requires secrets and is nondeterministic.
+
+Supporting reliability behavior such as judge/vision caching, URL redaction,
+image loading retries, and accessibility-tree filtering remains covered by the
+existing unit and extraction test suites rather than by presentation output.

example/site-with-errors/alt-text-errors.html

@@ -5,28 +5,76 @@
 ---
 
 <!--
-  Each <img> on this page intentionally uses bad alt text to exercise a single
-  rule in the accessibility-scanner-alt-text-plugin. See README.md in this
-  folder for the full image-to-rule mapping.
-
-  Every image points at the same throwaway SVG; the scanner only inspects the
-  `alt` attribute, not the image itself.
+  The first section exercises the five deterministic rules. The second section
+  supplies controlled inputs for the opt-in model rule; tests inject fixed
+  verdicts for those cases and never present them as live model output.
 -->
 
 <h1>Alt text errors</h1>
 
+<h2>Deterministic rules</h2>
+
 <!-- Rule: missing-alt-text — the alt attribute is absent entirely. -->
-<img src="/assets/img/test-image.svg" />
+<img data-expected-rule="missing-alt-text" src="/assets/img/test-image.svg" />
 
 <!-- Rule: placeholder-alt-text — "TODO" is boilerplate placeholder text. -->
-<img src="/assets/img/test-image.svg" alt="TODO" />
+<img data-expected-rule="placeholder-alt-text" src="/assets/img/test-image.svg" alt="TODO" />
 
 <!-- Rule: filename-alt-text — the alt text is a raw image filename. -->
-<img src="/assets/img/test-image.svg" alt="screenshot_2024.png" />
+<img data-expected-rule="filename-alt-text" src="/assets/img/test-image.svg" alt="screenshot_2024.png" />
 
 <!-- Rule: vague-alt-text — "image" says nothing about the content. -->
-<img src="/assets/img/test-image.svg" alt="image" />
+<img data-expected-rule="vague-alt-text" src="/assets/img/test-image.svg" alt="image" />
+
+<!-- Rule: repeated-alt-text — a tight visual group shares identical alt text. -->
+<span style="display: inline-flex; gap: 8px">
+  <img
+    data-expected-rule="repeated-alt-text"
+    src="/assets/img/test-image.svg"
+    alt="company logo"
+    width="48"
+    height="48"
+  />
+  <img
+    data-expected-rule="repeated-alt-text"
+    src="/assets/img/test-image.svg"
+    alt="company logo"
+    width="48"
+    height="48"
+  />
+</span>
+
+<h2>Opt-in model-backed quality cases</h2>
+
+<p>
+  These examples require a model verdict. The automated demo test uses fixed fake-judge responses so its evidence is
+  deterministic and does not call GitHub Models or Azure AI Vision.
+</p>
+
+<figure>
+  <img
+    data-model-case="keyword-stuffing"
+    src="/assets/img/test-image.svg"
+    alt="running shoes, cheap shoes, buy shoes online, best shoes 2026"
+  />
+  <figcaption>A keyword list that should be replaced with a concise description.</figcaption>
+</figure>
+
+<figure>
+  <img data-model-case="inaccurate" src="/assets/img/test-image.svg" alt="A team collaborating around a table" />
+  <figcaption>The image is actually a blue square containing the word test.</figcaption>
+</figure>
+
+<figure>
+  <img data-model-case="decorative" src="/assets/img/test-image.svg" alt="Blue divider pattern" />
+  <figcaption>The visible caption already communicates the decorative divider's purpose.</figcaption>
+</figure>
 
-<!-- Rule: repeated-alt-text — two consecutive images share identical alt text. -->
-<img src="/assets/img/test-image.svg" alt="company logo" />
-<img src="/assets/img/test-image.svg" alt="company logo" />
+<figure>
+  <img
+    data-model-case="accurate-control"
+    src="/assets/img/test-image.svg"
+    alt="A blue square with the word test in white"
+  />
+  <figcaption>An accurate control that should not produce a quality finding.</figcaption>
+</figure>

package.json

@@ -31,7 +31,9 @@
     "lint": "eslint .",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
-    "grade": "tsx --env-file=.env scripts/grade-alt-text-quality.ts"
+    "grade": "tsx --env-file=.env scripts/grade-alt-text-quality.ts",
+    "demo:verify": "tsx scripts/verify-demo.ts",
+    "demo:live": "tsx scripts/run-live-demo.ts"
   },
   "prettier": "@github/prettier-config",
   "engines": {