diff --git a/.github/workflows/scan-demo-site.yml b/.github/workflows/scan-demo-site.yml
new file mode 100644
index 0000000..36ba68a
--- /dev/null
+++ b/.github/workflows/scan-demo-site.yml
@@ -0,0 +1,108 @@
+name: Scan controlled demo site
+
+on:
+ workflow_dispatch:
+ inputs:
+ run_live_quality:
+ description: Run the optional credentialed GitHub Models quality demo
+ required: false
+ default: false
+ type: boolean
+ pull_request:
+ paths:
+ - .github/workflows/scan-demo-site.yml
+ - example/site-with-errors/**
+
+permissions:
+ contents: read
+
+jobs:
+ scan:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v7
+
+ - name: Setup Ruby
+ uses: ruby/setup-ruby@v1
+ with:
+ ruby-version: '3.4'
+ bundler-cache: true
+ working-directory: example/site-with-errors
+
+ - name: Setup Node
+ uses: actions/setup-node@v7
+ with:
+ node-version: 24
+ cache: npm
+
+ - name: Install plugin dependencies
+ run: npm ci
+
+ - name: Install Playwright browser
+ run: npx playwright install --with-deps chromium
+
+ - name: Build demo site
+ working-directory: example/site-with-errors
+ run: bundle exec jekyll build
+
+ - name: Serve demo site
+ run: |
+ python3 -m http.server 4000 --directory example/site-with-errors/_site > "$RUNNER_TEMP/demo-site.log" 2>&1 &
+ echo "$!" > "$RUNNER_TEMP/demo-site.pid"
+
+ for attempt in {1..10}; do
+ if curl --fail --silent --show-error http://127.0.0.1:4000/alt-text-errors/ > /dev/null; then
+ exit 0
+ fi
+ sleep 1
+ done
+
+ cat "$RUNNER_TEMP/demo-site.log"
+ exit 1
+
+ - name: Produce deterministic demo evidence
+ run: npm run --silent demo:verify > "$RUNNER_TEMP/demo-evidence.json"
+
+ - name: Run optional live quality demo
+ if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_live_quality }}
+ env:
+ GITHUB_MODELS_TOKEN: ${{ secrets.GH_MODELS_TOKEN }}
+ AZURE_VISION_ENDPOINT: ${{ secrets.AZURE_VISION_ENDPOINT }}
+ AZURE_VISION_KEY: ${{ secrets.AZURE_VISION_KEY }}
+ run: npm run --silent demo:live > "$RUNNER_TEMP/live-demo-evidence.json"
+
+ - name: Run accessibility scanner
+ id: scanner
+ uses: github/accessibility-scanner@745198705c4aecb2af83732eaafe1f6e3f7787c2 # v3.4.1
+ with:
+ urls: http://127.0.0.1:4000/alt-text-errors/
+ cache_key: cached_findings-controlled-demo-${{ github.run_id }}.json
+ repository: ${{ github.repository }}
+ token: ${{ github.token }}
+ skip_copilot_assignment: true
+ dry_run: true
+ scans: |
+ ["axe", {"name": "alt-text-scan", "package": "@github/accessibility-scanner-alt-text-plugin", "version": "1.1.0"}]
+
+ - name: Stage demo evidence
+ if: ${{ always() && steps.scanner.outputs.results_file }}
+ run: |
+ mkdir -p "$RUNNER_TEMP/controlled-demo-evidence"
+ cp "${{ steps.scanner.outputs.results_file }}" "$RUNNER_TEMP/controlled-demo-evidence/scanner-results.json"
+ cp "$RUNNER_TEMP/demo-evidence.json" "$RUNNER_TEMP/controlled-demo-evidence/demo-evidence.json"
+ if [[ -f "$RUNNER_TEMP/live-demo-evidence.json" ]]; then
+ cp "$RUNNER_TEMP/live-demo-evidence.json" "$RUNNER_TEMP/controlled-demo-evidence/live-demo-evidence.json"
+ fi
+
+ - name: Upload scanner results
+ if: ${{ always() && steps.scanner.outputs.results_file }}
+ uses: actions/upload-artifact@v7
+ with:
+ name: controlled-demo-results-${{ github.run_id }}
+ path: ${{ runner.temp }}/controlled-demo-evidence
+ if-no-files-found: error
+
+ - name: Stop demo site
+ if: ${{ always() }}
+ run: kill "$(cat "$RUNNER_TEMP/demo-site.pid")"
diff --git a/example/site-with-errors/README.md b/example/site-with-errors/README.md
index 0a1138a..922e567 100644
--- a/example/site-with-errors/README.md
+++ b/example/site-with-errors/README.md
@@ -14,10 +14,10 @@ Use it for:
suite (see [`tests/example-site.test.ts`](../../tests/example-site.test.ts)),
so the rules stay exercised against real markup in CI.
-## Image → rule mapping
+## Deterministic image → rule mapping
-Every image on [`alt-text-errors.html`](alt-text-errors.html) points at the same
-placeholder SVG (`assets/img/test-image.svg`); only the `alt` attribute differs.
+The first section of [`alt-text-errors.html`](alt-text-errors.html) exercises
+each rule that runs by default in plugin v1.1.0.
| Image `alt` value | Rule triggered | Why it triggers |
| ---------------------------- | ---------------------- | ------------------------------------------------ |
@@ -27,6 +27,57 @@ placeholder SVG (`assets/img/test-image.svg`); only the `alt` attribute differs.
| `image` | `vague-alt-text` | A single generic word that describes nothing. |
| `company logo` (×2 in a row) | `repeated-alt-text` | Two consecutive images share identical alt text. |
+These are real, credential-free plugin findings. The dedicated
+[`scan-demo-site.yml`](../../.github/workflows/scan-demo-site.yml) workflow
+builds this Jekyll site, serves only `/alt-text-errors/`, and runs Scanner
+v3.4.1 with Axe plus the npm-published plugin v1.1.0. It uses scanner dry-run
+mode and uploads `scanner-results.json`, so the workflow proves npm
+installation and execution without writing issues.
+
+## Model-backed quality cases
+
+The second section contains four inputs for the opt-in `alt-text-quality` rule:
+
+| Case | Expected mocked verdict | Evidence shown |
+| ---------------- | ----------------------- | ------------------------------------ |
+| Keyword stuffing | `needs-fix` | Tailored SEO-abuse finding |
+| Inaccurate alt | `needs-fix` | Finding with a suggested replacement |
+| Decorative image | `decorative` | Recommendation to use `alt=""` |
+| Accurate control | `ok` | No finding |
+
+These outcomes are **mocked test evidence**, not live model results. The
+targeted test injects fixed judge verdicts, then runs the production
+`alt-text-quality` rule-to-finding mapping. This keeps the demo deterministic
+and credential-free while clearly showing behavior that would otherwise
+require GitHub Models and, optionally, Azure AI Vision.
+
+The credential-free verifier prints all demo evidence as JSON:
+
+```sh
+npm run demo:verify
+```
+
+Its output separates:
+
+- the real deterministic plugin findings,
+- mocked quality verdicts using real context extraction, remediation mapping,
+ and scanner finding emission, and
+- mocked Azure caption, OCR, and tag signals passed through the production
+ Azure context-enrichment layer.
+
+For an optional live run, set `GITHUB_MODELS_TOKEN` to a PAT with `models:read`
+and run:
+
+```sh
+npm run demo:live
+```
+
+If `AZURE_VISION_ENDPOINT` and `AZURE_VISION_KEY` are also set, the live command
+automatically requests Azure-augmented mode. Set
+`ALT_TEXT_JUDGE_MODE=copilot` or `ALT_TEXT_JUDGE_MODE=azure-augmented` to force
+a mode. Live output is credentialed and nondeterministic; it is not part of the
+required CI evidence.
+
## Run it locally
The site is a standard Jekyll site served as a static build behind Rack/Puma.
@@ -54,12 +105,42 @@ You don't need Ruby or a running server to confirm the plugin flags this page.
From the repository root:
```sh
-npm install
+npm ci
npx playwright install chromium
-npm test
+npm test -- tests/example-site.test.ts tests/unit/azure-augmented-judge.test.ts --reporter=verbose
```
-The `example site-with-errors` test loads
+The targeted tests load
[`alt-text-errors.html`](alt-text-errors.html), runs the real `alt-text-scan`
-plugin against it, and asserts that every rule in the table above produces a
-finding.
+plugin against it, asserts exactly one finding for each deterministic rule,
+checks the four model cases through fixed fake-judge verdicts, and exercise the
+production Azure enrichment and fallback layers with a fake Azure client. No
+model or Azure credentials are used.
+
+## Evidence and limitations
+
+| Feature | Evidence layer | Expected result |
+| -------------------------------- | ------------------------------ | --------------------------------------------------------------------------- |
+| Scanner v3.4.1 action | Hosted workflow | Immutable scanner commit is downloaded and the scan step succeeds. |
+| npm plugin loading | Hosted workflow | npm installs v1.1.0, then Scanner discovers and runs `alt-text-scan`. |
+| Five deterministic rules | Hosted artifact and local test | Exactly one plugin finding for each rule. |
+| Rule configuration | Local test | Disabling `missing-alt-text` suppresses only that finding. |
+| Context extraction | Local test | Model input includes the page title, nearest heading, and figure caption. |
+| Keyword stuffing | Fixed fake judge | Production mapping emits the tailored SEO-abuse finding. |
+| Inaccurate alt and remediation | Fixed fake judge | Production mapping emits an `inaccurate` finding and suggested replacement. |
+| Decorative and accurate controls | Fixed fake judge | Decorative yields an `alt=""` recommendation; accurate yields no finding. |
+| Azure caption, OCR, and tags | Fake Azure client | Production enrichment adds high-confidence signals to model context. |
+| Azure failure fallback | Fake Azure client | Production enrichment falls back to unmodified Copilot-only context. |
+| Axe alongside the plugin | Hosted artifact | Axe findings and plugin findings appear in the same scanner results file. |
+
+The hosted Scanner action proves only behavior available through the published
+npm package without model credentials: npm loading, Axe, and the five default
+rules. Scanner cannot inject the repository's fake judge into the published
+package, so model and Azure evidence comes from the checked-in v1.1.0 source
+through `demo:verify` and targeted tests. The optional `demo:live` path is the
+only evidence that calls GitHub Models or Azure; its output is intentionally
+not asserted in CI because it requires secrets and is nondeterministic.
+
+Supporting reliability behavior such as judge/vision caching, URL redaction,
+image loading retries, and accessibility-tree filtering remains covered by the
+existing unit and extraction test suites rather than by presentation output.
diff --git a/example/site-with-errors/alt-text-errors.html b/example/site-with-errors/alt-text-errors.html
index c3488ce..313f726 100644
--- a/example/site-with-errors/alt-text-errors.html
+++ b/example/site-with-errors/alt-text-errors.html
@@ -5,28 +5,76 @@
---
Alt text errors
+
Deterministic rules
+
-
+
-
+
-
+
-
+
+
+
+
+
+
+
+
+
Opt-in model-backed quality cases
+
+
+ These examples require a model verdict. The automated demo test uses fixed fake-judge responses so its evidence is
+ deterministic and does not call GitHub Models or Azure AI Vision.
+
+
+
+
+ A keyword list that should be replaced with a concise description.
+
+
+
+
+ The image is actually a blue square containing the word test.
+
+
+
+
+ The visible caption already communicates the decorative divider's purpose.
+
-
-
-
+
+
+ An accurate control that should not produce a quality finding.
+
diff --git a/package.json b/package.json
index 96f4953..5184614 100644
--- a/package.json
+++ b/package.json
@@ -31,7 +31,9 @@
"lint": "eslint .",
"format": "prettier --write .",
"format:check": "prettier --check .",
- "grade": "tsx --env-file=.env scripts/grade-alt-text-quality.ts"
+ "grade": "tsx --env-file=.env scripts/grade-alt-text-quality.ts",
+ "demo:verify": "tsx scripts/verify-demo.ts",
+ "demo:live": "tsx scripts/run-live-demo.ts"
},
"prettier": "@github/prettier-config",
"engines": {
diff --git a/scripts/demo-support.ts b/scripts/demo-support.ts
new file mode 100644
index 0000000..b2e619e
--- /dev/null
+++ b/scripts/demo-support.ts
@@ -0,0 +1,77 @@
+import {readFile} from 'node:fs/promises'
+import {createServer, type Server} from 'node:http'
+import {chromium, type Browser, type Page} from 'playwright'
+
+const fixtureRoot = new URL('../example/site-with-errors/', import.meta.url)
+const routes = new Map([
+ ['/demo', {file: 'alt-text-errors.html', contentType: 'text/html; charset=utf-8'}],
+ ['/assets/img/test-image.svg', {file: 'assets/img/test-image.svg', contentType: 'image/svg+xml'}],
+])
+
+function stripFrontMatter(contents: Buffer): Buffer {
+ const text = contents.toString('utf8')
+ return Buffer.from(text.replace(/^---\n[\s\S]*?\n---\n/, ''), 'utf8')
+}
+
+async function startServer(): Promise<{server: Server; origin: string}> {
+ const server = createServer(async (request, response) => {
+ const route = routes.get(request.url ?? '')
+ if (!route) {
+ response.writeHead(404).end()
+ return
+ }
+
+ try {
+ const rawContents = await readFile(new URL(route.file, fixtureRoot))
+ const contents = route.contentType.startsWith('text/html') ? stripFrontMatter(rawContents) : rawContents
+ response.writeHead(200, {'Content-Type': route.contentType})
+ response.end(contents)
+ } catch (error) {
+ response.writeHead(500, {'Content-Type': 'text/plain; charset=utf-8'})
+ response.end(error instanceof Error ? error.message : String(error))
+ }
+ })
+
+ await new Promise((resolve, reject) => {
+ server.once('error', reject)
+ server.listen(0, '127.0.0.1', resolve)
+ })
+
+ const address = server.address()
+ if (!address || typeof address === 'string') throw new Error('Demo server did not bind to a TCP port.')
+ return {server, origin: `http://127.0.0.1:${address.port}`}
+}
+
+export type DemoHarness = {
+ page: Page
+ open(): Promise
+ close(): Promise
+}
+
+export async function createDemoHarness(): Promise {
+ const {server, origin} = await startServer()
+ let browser: Browser
+ try {
+ browser = await chromium.launch()
+ } catch (error) {
+ server.close()
+ throw error
+ }
+ const page = await browser.newPage()
+
+ return {
+ page,
+ async open() {
+ const url = `${origin}/demo`
+ await page.goto(url)
+ return url
+ },
+ async close() {
+ await page.close()
+ await browser.close()
+ await new Promise((resolve, reject) => {
+ server.close(error => (error ? reject(error) : resolve()))
+ })
+ },
+ }
+}
diff --git a/scripts/run-live-demo.ts b/scripts/run-live-demo.ts
new file mode 100644
index 0000000..f7a07a2
--- /dev/null
+++ b/scripts/run-live-demo.ts
@@ -0,0 +1,52 @@
+import {readFile} from 'node:fs/promises'
+import {emitFindings} from '../src/findings.js'
+import {extractImageContext} from '../src/extract-image-context.js'
+import {__setJudge, altTextQuality} from '../src/rules/alt-text-quality.js'
+import type {Finding} from '../src/types.js'
+import {createDemoHarness} from './demo-support.js'
+
+if (!process.env['GITHUB_MODELS_TOKEN'] && !process.env['GITHUB_TOKEN']) {
+ throw new Error('Set GITHUB_MODELS_TOKEN to a PAT with models:read before running the live demo.')
+}
+
+const azureConfigured = Boolean(process.env['AZURE_VISION_ENDPOINT'] && process.env['AZURE_VISION_KEY'])
+const requestedMode = process.env['ALT_TEXT_JUDGE_MODE']
+const judgeMode =
+ requestedMode === 'copilot' || requestedMode === 'azure-augmented'
+ ? requestedMode
+ : azureConfigured
+ ? 'azure-augmented'
+ : 'copilot'
+
+const harness = await createDemoHarness()
+try {
+ const fixtureUrl = await harness.open()
+ const images = (await extractImageContext(harness.page)).filter(image => image.outerHTML.includes('data-model-case='))
+ __setJudge(null)
+ const results = await altTextQuality.evaluate({url: fixtureUrl, images})
+ const findings: Finding[] = []
+ await emitFindings(altTextQuality, results, fixtureUrl, async finding => {
+ findings.push(finding)
+ })
+
+ const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')) as {
+ version: string
+ }
+ console.log(
+ JSON.stringify(
+ {
+ pluginVersion: packageJson.version,
+ evidence: 'live GitHub Models evaluation of the controlled baseline fixture',
+ judgeMode,
+ azureCredentialsConfigured: azureConfigured,
+ fixtureUrl,
+ findings,
+ },
+ null,
+ 2,
+ ),
+ )
+} finally {
+ __setJudge(null)
+ await harness.close()
+}
diff --git a/scripts/verify-demo.ts b/scripts/verify-demo.ts
new file mode 100644
index 0000000..c04b75a
--- /dev/null
+++ b/scripts/verify-demo.ts
@@ -0,0 +1,142 @@
+import {readFile} from 'node:fs/promises'
+import altTextScan from '../index.js'
+import {emitFindings} from '../src/findings.js'
+import {extractImageContext} from '../src/extract-image-context.js'
+import {
+ createAzureAugmentedJudge,
+ type AzureVisionClient,
+ type JudgeAltText,
+ type JudgeInput,
+ type JudgeVerdict,
+} from '../src/judges/index.js'
+import {__setJudge, altTextQuality} from '../src/rules/alt-text-quality.js'
+import type {Finding} from '../src/types.js'
+import {createDemoHarness} from './demo-support.js'
+
+class DemoJudge implements JudgeAltText {
+ async judge(input: JudgeInput): Promise {
+ switch (input.alt) {
+ case 'running shoes, cheap shoes, buy shoes online, best shoes 2026':
+ return {
+ step: 4,
+ reasoning: 'This is a keyword list rather than an image description.',
+ verdict: 'needs-fix',
+ issue: 'keyword-stuffing',
+ confidence: 1,
+ suggestion: 'Blue square with the word test',
+ }
+ case 'A team collaborating around a table':
+ return {
+ step: 4,
+ reasoning: 'The alt text does not match the controlled image.',
+ verdict: 'needs-fix',
+ issue: 'inaccurate',
+ confidence: 1,
+ suggestion: 'Blue square with the word test',
+ }
+ case 'Blue divider pattern':
+ return {
+ step: 4,
+ reasoning: 'The visible caption already communicates its purpose.',
+ verdict: 'decorative',
+ issue: '',
+ confidence: 1,
+ suggestion: '',
+ }
+ case 'A blue square with the word test in white':
+ return {
+ step: 4,
+ reasoning: 'The alt text accurately describes the controlled image.',
+ verdict: 'ok',
+ issue: '',
+ confidence: 1,
+ suggestion: '',
+ }
+ default:
+ throw new Error(`Unexpected model demo input: ${input.alt}`)
+ }
+ }
+}
+
+class CapturingJudge implements JudgeAltText {
+ input: JudgeInput | null = null
+
+ async judge(input: JudgeInput): Promise {
+ this.input = input
+ return {step: 4, reasoning: 'Accurate.', verdict: 'ok', issue: '', confidence: 1, suggestion: ''}
+ }
+}
+
+const mockedVision: AzureVisionClient = {
+ async analyze() {
+ return {
+ caption: {text: 'a blue square', confidence: 0.99},
+ readText: 'test',
+ tags: [
+ {name: 'graphic', confidence: 0.95},
+ {name: 'low-confidence-noise', confidence: 0.1},
+ ],
+ }
+ },
+}
+
+const harness = await createDemoHarness()
+try {
+ const fixtureUrl = await harness.open()
+ const deterministicFindings: Finding[] = []
+ await altTextScan({
+ page: harness.page,
+ addFinding: async finding => {
+ deterministicFindings.push(finding)
+ },
+ })
+
+ const images = await extractImageContext(harness.page)
+ const modelCases = images.filter(image => image.outerHTML.includes('data-model-case='))
+ __setJudge(new DemoJudge())
+ const results = await altTextQuality.evaluate({url: fixtureUrl, images: modelCases})
+ const mockedModelFindings: Finding[] = []
+ await emitFindings(altTextQuality, results, fixtureUrl, async finding => {
+ mockedModelFindings.push(finding)
+ })
+
+ const capturingJudge = new CapturingJudge()
+ const azureJudge = createAzureAugmentedJudge({inner: capturingJudge, vision: mockedVision})
+ await azureJudge.judge({
+ imageDataUrl: 'data:image/png;base64,iVBORw0KGgo=',
+ alt: 'A blue square with the word test in white',
+ context: 'Controlled demo page context.',
+ naturalWidth: 120,
+ naturalHeight: 120,
+ })
+ const enrichedContext = capturingJudge.input?.context ?? ''
+ const expectedAzureSignals = ['Azure CV caption: a blue square', 'Azure CV OCR: test', 'Azure CV tags: graphic']
+ for (const signal of expectedAzureSignals) {
+ if (!enrichedContext.includes(signal)) throw new Error(`Missing mocked Azure evidence: ${signal}`)
+ }
+
+ const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')) as {
+ version: string
+ }
+ console.log(
+ JSON.stringify(
+ {
+ pluginVersion: packageJson.version,
+ fixtureUrl,
+ evidence: {
+ deterministic: 'real plugin scan; no credentials or model calls',
+ modelBacked: 'fixed fake judge; real extraction, rule mapping, remediation, and Finding shape',
+ azureAugmentation: 'fixed fake Azure client; real context-enrichment decorator',
+ },
+ deterministicFindings,
+ mockedModelFindings,
+ mockedAzureSignals: expectedAzureSignals,
+ },
+ null,
+ 2,
+ ),
+ )
+} finally {
+ __setJudge(null)
+ await harness.close()
+}
diff --git a/tests/example-site.test.ts b/tests/example-site.test.ts
index 859803b..d83d71d 100644
--- a/tests/example-site.test.ts
+++ b/tests/example-site.test.ts
@@ -3,10 +3,70 @@ import {fileURLToPath} from 'node:url'
import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest'
import {chromium, type Browser, type Page} from 'playwright'
import altTextScan from '../index.js'
-import type {Finding} from '../src/types.js'
+import {extractImageContext} from '../src/extract-image-context.js'
+import type {JudgeAltText, JudgeInput, JudgeVerdict} from '../src/judges/index.js'
+import {altTextQuality, __setJudge} from '../src/rules/alt-text-quality.js'
+import type {Finding, RuleResult} from '../src/types.js'
const errorsPagePath = fileURLToPath(new URL('../example/site-with-errors/alt-text-errors.html', import.meta.url))
const fixtureWithDisabledRule = fileURLToPath(new URL('./fixtures/with-disabled-rule', import.meta.url))
+const DATA_URL = 'data:image/png;base64,iVBORw0KGgo='
+const deterministicRuleIds = [
+ 'missing-alt-text',
+ 'placeholder-alt-text',
+ 'filename-alt-text',
+ 'vague-alt-text',
+ 'repeated-alt-text',
+] as const
+
+class FakeJudge implements JudgeAltText {
+ readonly calls: JudgeInput[] = []
+
+ async judge(input: JudgeInput): Promise {
+ this.calls.push(input)
+
+ switch (input.alt) {
+ case 'running shoes, cheap shoes, buy shoes online, best shoes 2026':
+ return {
+ step: 4,
+ reasoning: 'This is a keyword list rather than an image description.',
+ verdict: 'needs-fix',
+ issue: 'keyword-stuffing',
+ confidence: 0.99,
+ suggestion: 'Blue square with the word test',
+ }
+ case 'A team collaborating around a table':
+ return {
+ step: 4,
+ reasoning: 'The alt text does not match the controlled image.',
+ verdict: 'needs-fix',
+ issue: 'inaccurate',
+ confidence: 0.99,
+ suggestion: 'Blue square with the word test',
+ }
+ case 'Blue divider pattern':
+ return {
+ step: 4,
+ reasoning: 'The visible caption already communicates its purpose.',
+ verdict: 'decorative',
+ issue: '',
+ confidence: 0.99,
+ suggestion: '',
+ }
+ case 'A blue square with the word test in white':
+ return {
+ step: 4,
+ reasoning: 'The alt text accurately describes the controlled image.',
+ verdict: 'ok',
+ issue: '',
+ confidence: 0.99,
+ suggestion: '',
+ }
+ default:
+ throw new Error(`Unexpected model demo input: ${input.alt}`)
+ }
+ }
+}
// Strips the Jekyll/Liquid front matter so the raw markup can be loaded
// directly into Playwright without running a Jekyll build.
@@ -31,11 +91,12 @@ beforeEach(async () => {
})
afterEach(async () => {
+ __setJudge(null)
await page.close()
})
describe('example site-with-errors', () => {
- it('produces a finding for every alt-text rule', async () => {
+ it('produces exactly one real finding for each deterministic fixture case', async () => {
const body = loadErrorsPageBody()
await page.setContent(`${body}`)
@@ -47,15 +108,50 @@ describe('example site-with-errors', () => {
},
})
- const ruleIds = new Set(findings.map(f => f.ruleId))
-
- const {allRules} = await import('../src/rules/index.js')
- for (const rule of allRules) {
- if (rule.defaultEnabled === false) continue
- expect(ruleIds).toContain(rule.id)
+ expect(findings).toHaveLength(deterministicRuleIds.length)
+ for (const ruleId of deterministicRuleIds) {
+ const matching = findings.filter(finding => finding.ruleId === ruleId)
+ expect(matching).toHaveLength(1)
+ expect(matching[0]!.html).toContain(`data-expected-rule="${ruleId}"`)
}
})
+ it('maps model-backed fixture cases through deterministic mocked verdicts', async () => {
+ const body = loadErrorsPageBody()
+ await page.setContent(`Alt text demo${body}`)
+
+ const images = await extractImageContext(page)
+ const modelCases = images
+ .filter(image => image.outerHTML.includes('data-model-case='))
+ .map(image => ({...image, src: DATA_URL}))
+ expect(modelCases).toHaveLength(4)
+
+ const fakeJudge = new FakeJudge()
+ __setJudge(fakeJudge)
+ const results = (await altTextQuality.evaluate({
+ url: 'https://example.test/alt-text-errors/',
+ images: modelCases,
+ })) as RuleResult[]
+
+ expect(fakeJudge.calls).toHaveLength(4)
+ expect(results).toHaveLength(3)
+ expect(fakeJudge.calls.every(call => call.context.includes('Page title: "Alt text demo"'))).toBe(true)
+ expect(fakeJudge.calls.every(call => call.context.includes('Nearest heading above the image'))).toBe(true)
+ expect(fakeJudge.calls.every(call => call.context.includes('Adjacent figcaption'))).toBe(true)
+
+ const keywordStuffing = results.find(result => result.image.alt?.startsWith('running shoes'))
+ expect(keywordStuffing?.problemShort).toContain('keyword-stuffed')
+
+ const inaccurate = results.find(result => result.image.alt === 'A team collaborating around a table')
+ expect(inaccurate?.problemShort).toContain('inaccurate')
+ expect(inaccurate?.solutionShort).toContain('Blue square with the word test')
+
+ const decorative = results.find(result => result.image.alt === 'Blue divider pattern')
+ expect(decorative?.solutionShort).toContain('alt=""')
+
+ expect(results.some(result => result.image.alt === 'A blue square with the word test in white')).toBe(false)
+ })
+
it('produces no findings for an image with valid alt text', async () => {
await page.setContent(
``,
diff --git a/tests/unit/azure-augmented-judge.test.ts b/tests/unit/azure-augmented-judge.test.ts
new file mode 100644
index 0000000..335ad82
--- /dev/null
+++ b/tests/unit/azure-augmented-judge.test.ts
@@ -0,0 +1,75 @@
+import {describe, expect, it, vi} from 'vitest'
+import {
+ createAzureAugmentedJudge,
+ type AzureVisionClient,
+ type JudgeAltText,
+ type JudgeInput,
+ type JudgeVerdict,
+} from '../../src/judges/index.js'
+
+class CapturingJudge implements JudgeAltText {
+ readonly inputs: JudgeInput[] = []
+
+ async judge(input: JudgeInput): Promise {
+ this.inputs.push(input)
+ return {step: 4, reasoning: 'Accurate.', verdict: 'ok', issue: '', confidence: 1, suggestion: ''}
+ }
+}
+
+function input(overrides: Partial = {}): JudgeInput {
+ return {
+ imageDataUrl: 'data:image/png;base64,iVBORw0KGgo=',
+ alt: 'A blue square with the word test',
+ context: 'Original page context.',
+ naturalWidth: 120,
+ naturalHeight: 120,
+ ...overrides,
+ }
+}
+
+describe('createAzureAugmentedJudge', () => {
+ it('adds mocked caption, OCR, and high-confidence tags to the model context', async () => {
+ const inner = new CapturingJudge()
+ const vision: AzureVisionClient = {
+ async analyze() {
+ return {
+ caption: {text: 'a blue square', confidence: 0.99},
+ denseCaptions: [{text: 'white text centered in a blue box', confidence: 0.95}],
+ readText: 'test',
+ tags: [
+ {name: 'graphic', confidence: 0.95},
+ {name: 'low-confidence-noise', confidence: 0.1},
+ ],
+ }
+ },
+ }
+
+ const judge = createAzureAugmentedJudge({inner, vision})
+ await expect(judge.judge(input())).resolves.toMatchObject({verdict: 'ok'})
+
+ const context = inner.inputs[0]!.context
+ expect(context).toContain('Original page context.')
+ expect(context).toContain('Azure CV caption: a blue square')
+ expect(context).toContain('Azure CV regions: white text centered in a blue box')
+ expect(context).toContain('Azure CV OCR: test')
+ expect(context).toContain('Azure CV tags: graphic')
+ expect(context).not.toContain('low-confidence-noise')
+ })
+
+ it('falls back to the original context when the mocked Azure pre-pass fails', async () => {
+ const inner = new CapturingJudge()
+ const vision: AzureVisionClient = {
+ async analyze() {
+ throw new Error('mock Azure outage')
+ },
+ }
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const judge = createAzureAugmentedJudge({inner, vision})
+ await expect(judge.judge(input())).resolves.toMatchObject({verdict: 'ok'})
+
+ expect(inner.inputs[0]!.context).toBe('Original page context.')
+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('falling back to Copilot-only'))
+ warn.mockRestore()
+ })
+})
+