diff --git a/README.md b/README.md index fdde74f..2ff5dd0 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ TypeScript, installed as executables on `PATH`. | [`genrewatch`](#genrewatch) | What is coming out, and whether it exists at all | | [`img`](#img) | Resize, convert and inspect images, with sharp or ImageMagick | | [`favicon`](#favicon) | Every icon a site links, rendered from one SVG | +| [`wcag`](#wcag) | Audit a site against WCAG with axe in headless Chrome, for the W3C report tool | | [`vid`](#vid) | Inspect, thumbnail, clip and shrink video, through ffmpeg | | [`dl`](#dl) | Download a video, or just its audio, through yt-dlp | | [`torrent`](#torrent) | Make a torrent out of a directory, and get it seeded | @@ -59,6 +60,9 @@ One thing here is not a `PATH` command and does not need Node: [torlnk](https://www.npmjs.com/package/torlnk) running - **ImageMagick** (`magick`) — `img` only, and only for what sharp cannot do (PDF, PSD, animated GIF); sharp ships with this repo as an optional dependency +- **A Chrome or Chromium** — `wcag` only. `CHROME_PATH` names one; otherwise + the usual binaries on `PATH` are tried, then the builds Puppeteer and + Playwright keep under `~/.cache`. axe-core itself ships with this repo - **`unzip` or `bsdtar`** — the `adb` companion only, and only while installing it: the archive Google publishes is a zip, and Node cannot read one - **Network on first use** — `favicon` only: the generation is @@ -1064,6 +1068,58 @@ and `-i`/`-o` are always passed on, so that prompt is unreachable. when a release breaks you, or point it at a checkout while working on the generator itself. +### `wcag` + +The automated half of a WCAG-EM evaluation, with the W3C's +[WCAG-EM Report Tool](https://www.w3.org/WAI/eval/report-tool/) for the other +half: + +```sh +wcag audit https://example.org # 5 pages from the sitemap, WCAG 2.2 AA +wcag audit https://example.org --pages 12 --level AAA --md audit.md +wcag audit https://example.org --sample list --url https://example.org/checkout +wcag report wcag-report.json # -> evaluation.json, for the report tool +wcag open # the hand-off, step by step +``` + +The report tool is a web page with no command line. It is a form for the +five steps of the methodology, and the reason it cannot be a CLI is that most +success criteria need a person to decide them. What a machine can do is the +part that is machine-shaped, and that is what `audit` does: choose the +structured sample (step 3 — the home page, then one page per section of the +sitemap before a second of any, so a blog gives up its about, pricing and docs +pages before a second post), load each in headless Chrome, run +[axe-core](https://github.com/dequelabs/axe-core) over it, and print one row +per success criterion: how many pages fail it, how many need a look, and the +rule with the most elements in violation. `report` turns that into the tool's +own evaluation file, which "Open evaluation" loads with the scope, the sample +and one assertion per page and criterion filled in, so the evaluator starts +at the judgement calls rather than at an empty form. + +**Nothing is ever marked passed.** axe can prove a failure — an image with no +alternative fails 1.1.1 wherever it is — but "no rule fired" proves nothing +about a criterion as a whole, since its rules cover a part of each one. A +criterion with only passing checks lands in the tool as "cannot tell", with +the checks listed; the summary shows the passes in their own column because +they are still worth seeing. The exit status is 1 when a criterion within the +target fails on any page, which is what a CI step wants to know. + +The browser is driven over the DevTools protocol by hand — Node's own +WebSocket, six protocol methods — rather than through Puppeteer or Playwright, +which would each bring a Chrome download and a driver to open a page and +evaluate two scripts. The box's Chrome is found instead: `CHROME_PATH`, then +`PATH`, then the builds those two keep in `~/.cache`. A Puppeteer build on a +box without Chrome's system libraries runs when they are staged under +`~/.local/share/chrome-deps`, and that is wired up here so the command works +the same whichever Chrome it finds. `CHROME_NO_SANDBOX=1` drops the sandbox for +containers and root. + +The evaluation file mirrors the tool's own export, read from its source: the +JSON-LD context, one `Webpage` subject per sampled page whose id is its URL, +and `Assertion`s whose `test` is the tool's own criterion id +(`WCAG22:contrast-minimum`). `--wcag 2.1` uses the 2.1 ids and leaves the nine +2.2-only criteria out. + ### `vid` The four things anybody actually needs ffmpeg for: diff --git a/bin/wcag.ts b/bin/wcag.ts new file mode 100644 index 0000000..dad2e04 --- /dev/null +++ b/bin/wcag.ts @@ -0,0 +1,247 @@ +#!/usr/bin/env node +/** + * wcag — the automated half of a WCAG-EM evaluation, from the terminal. + * + * wcag audit https://example.org sample 5 pages, run axe, print the criteria + * wcag audit https://example.org --pages 12 --level AAA --md audit.md + * wcag report wcag-report.json the W3C report tool's evaluation file + * wcag open how to load it into the tool + * + * The W3C WCAG-EM Report Tool has no CLI; src/wcag.ts says what this does + * instead and why nothing here is ever marked "passed". + */ + +import { writeFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; + +import { UsageError, integer, parseArgs } from '../src/args.ts'; +import { isMain } from '../src/is-main.ts'; +import { + DEFAULT_LEVEL, + DEFAULT_VERSION, + NO_CHROME, + OPEN_STEPS, + REPORT_TOOL_URL, + type SampleMethod, + WcagError, + audit, + discoverSample, + findChrome, + formatSummary, + hasFailures, + isLevel, + isReport, + isSampleMethod, + isWcagVersion, + launchBrowser, + summarize, + toEvaluation, + toMarkdown, +} from '../src/wcag.ts'; + +const DEFAULT_PAGES = 5; +const DEFAULT_REPORT = 'wcag-report.json'; +const DEFAULT_EVALUATION = 'evaluation.json'; + +const USAGE = `Usage: + wcag audit [options] sample the site, run axe-core on each page in headless Chrome, + print one row per success criterion, write the report + wcag report [options] turn a report into the W3C WCAG-EM Report Tool's evaluation file + wcag open how to load that file into ${REPORT_TOOL_URL} + +Options for audit: + -n, --pages N how many pages to audit, including the start page (default: ${DEFAULT_PAGES}) + --sample METHOD auto | sitemap | links | list (default: auto — the sitemap, then the + start page's own links when the sitemap is short) + --url URL a page to include whatever the sample says; repeatable + --level A|AA|AAA the conformance target (default: ${DEFAULT_LEVEL}) + --wcag 2.1|2.2 which WCAG (default: ${DEFAULT_VERSION}) + -o, --out FILE where the report goes (default: ${DEFAULT_REPORT}) + --md FILE also write a Markdown summary + --timeout S seconds to give each page to load (default: 30) + --chrome PATH the browser to use (default: CHROME_PATH, then the usual places) + --json print the report to stdout instead of the table + --quiet no per-page progress on stderr + +Options for report: + -o, --out FILE where the evaluation goes (default: ${DEFAULT_EVALUATION}) + --site NAME the site's name in the report (default: its host) + --title T the evaluation's title + --evaluator NAME who is evaluating + --commissioner NAME who asked for it + + --help show this help + +Exit status is 1 when a criterion within the target fails on any page, 2 on +a usage error or when no Chrome can be found. A criterion with no failure has +not passed: axe covers a part of each one, and the rest is the evaluator's. + + CHROME_PATH a Chrome or Chromium binary, when the usual places have none + CHROME_NO_SANDBOX set to run Chrome without its sandbox (containers, root) +`; + +function fail(message: string, code = 2): never { + process.stderr.write(`wcag: ${message}\n`); + process.exit(code); +} + +function out(text: string): void { + process.stdout.write(text.endsWith('\n') ? text : `${text}\n`); +} + +function note(text: string): void { + process.stderr.write(`${text}\n`); +} + +/** `--url a --url b` is two values; parseArgs keeps the last, so they are read off argv directly. */ +function repeated(argv: readonly string[], flag: string): string[] { + const values: string[] = []; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]!; + if (argument === flag) { + const next = argv[index + 1]; + if (next !== undefined && !next.startsWith('-')) values.push(next); + } else if (argument.startsWith(`${flag}=`)) { + values.push(argument.slice(flag.length + 1)); + } + } + return values; +} + +function assertUrl(value: string): string { + let url: URL; + try { + url = new URL(value.includes('://') ? value : `https://${value}`); + } catch { + throw new UsageError(`not a URL: ${value}`); + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new UsageError(`only http and https pages can be audited, got ${url.protocol}`); + } + return url.href; +} + +async function runAudit(argv: string[]): Promise { + const { flags, values, positional } = parseArgs(argv, { + boolean: ['--json', '--quiet', '--help'], + string: ['-n', '--pages', '--sample', '--url', '--level', '--wcag', '-o', '--out', '--md', '--timeout', '--chrome'], + }); + if (flags.has('--help')) { + out(USAGE); + return 0; + } + if (positional.length !== 1) throw new UsageError('audit takes one URL: the page to start from'); + + const start = assertUrl(positional[0]!); + const pages = integer(values, values.has('-n') ? '-n' : '--pages', DEFAULT_PAGES, { min: 1, max: 200 }); + const method = values.get('--sample') ?? 'auto'; + if (!isSampleMethod(method)) throw new UsageError(`--sample must be auto, sitemap, links or list, got ${method}`); + const level = values.get('--level')?.toUpperCase() ?? DEFAULT_LEVEL; + if (!isLevel(level)) throw new UsageError(`--level must be A, AA or AAA, got ${level}`); + const version = values.get('--wcag') ?? DEFAULT_VERSION; + if (!isWcagVersion(version)) throw new UsageError(`--wcag must be 2.1 or 2.2, got ${version}`); + const timeoutMs = integer(values, '--timeout', 30, { min: 1, max: 600 }) * 1000; + const extra = repeated(argv, '--url').map(assertUrl); + const outFile = values.get('-o') ?? values.get('--out') ?? DEFAULT_REPORT; + const quiet = flags.has('--quiet') || flags.has('--json'); + + const chrome = values.get('--chrome') ?? findChrome(); + if (!chrome) throw new WcagError(NO_CHROME); + + if (!quiet) note(`sampling ${start} (${method})…`); + const sample = await discoverSample(start, { pages, method: method as SampleMethod, extra }); + if (!quiet) { + note(`${sample.pages.length} pages from ${sample.from === 'start' ? 'the start page alone' : `the ${sample.from}`}` + + (sample.candidates > sample.pages.length ? ` (${sample.candidates} candidates)` : '')); + } + + const browser = await launchBrowser({ chrome, timeoutMs }); + try { + const report = await audit(browser, sample, { + version, + level, + timeoutMs, + onPage: (result, index, total) => { + if (quiet) return; + const status = result.ok + ? `${result.violations.length} rules failing, ${result.incomplete.length} to review` + : `could not load: ${result.error ?? 'unknown'}`; + note(` [${index + 1}/${total}] ${result.url} — ${status} (${result.ms} ms)`); + }, + }); + + writeFileSync(outFile, `${JSON.stringify(report, null, 2)}\n`); + const markdown = values.get('--md'); + if (markdown) writeFileSync(markdown, toMarkdown(report)); + + if (flags.has('--json')) { + out(JSON.stringify(report, null, 2)); + } else { + out(formatSummary(report, summarize(report))); + out(`report: ${outFile}${markdown ? `, summary: ${markdown}` : ''}. Next: wcag report ${outFile}`); + } + return hasFailures(report) ? 1 : 0; + } finally { + await browser.close(); + } +} + +async function runReport(argv: string[]): Promise { + const { flags, values, positional } = parseArgs(argv, { + boolean: ['--help'], + string: ['-o', '--out', '--site', '--title', '--evaluator', '--commissioner'], + }); + if (flags.has('--help')) { + out(USAGE); + return 0; + } + if (positional.length !== 1) throw new UsageError('report takes one file: the report `wcag audit` wrote'); + + const file = positional[0]!; + let parsed: unknown; + try { + parsed = JSON.parse(await readFile(file, 'utf8')); + } catch (error) { + throw new WcagError(`cannot read ${file}: ${(error as Error).message}`); + } + if (!isReport(parsed)) throw new WcagError(`${file} is not a report written by \`wcag audit\``); + + const evaluation = toEvaluation(parsed, { + ...(values.has('--site') ? { site: values.get('--site')! } : {}), + ...(values.has('--title') ? { title: values.get('--title')! } : {}), + ...(values.has('--evaluator') ? { evaluator: values.get('--evaluator')! } : {}), + ...(values.has('--commissioner') ? { commissioner: values.get('--commissioner')! } : {}), + }); + const outFile = values.get('-o') ?? values.get('--out') ?? DEFAULT_EVALUATION; + writeFileSync(outFile, `${JSON.stringify(evaluation, null, 2)}\n`); + + const assertions = (evaluation.auditSample as unknown[]).length; + const sampled = ((evaluation.selectSample as { structuredSample: unknown[] }).structuredSample).length; + out(`${outFile}: ${sampled} pages in the sample, ${assertions} assertions. Open it at ${REPORT_TOOL_URL} with "Open evaluation".`); + return 0; +} + +if (isMain(import.meta.url)) { + const argv = process.argv.slice(2); + const verb = argv[0]; + try { + if (verb === undefined || verb === '--help' || verb === '-h' || verb === 'help') { + out(USAGE); + process.exit(verb === undefined ? 1 : 0); + } + let code: number; + if (verb === 'audit') code = await runAudit(argv.slice(1)); + else if (verb === 'report') code = await runReport(argv.slice(1)); + else if (verb === 'open') { + out(OPEN_STEPS); + code = 0; + } else throw new UsageError(`unknown command: ${verb}`); + process.exit(code); + } catch (error) { + if (error instanceof UsageError) { + process.stderr.write(`wcag: ${error.message}\n\n${USAGE}`); + process.exit(2); + } + fail((error as Error).message, error instanceof WcagError ? 2 : 1); + } +} diff --git a/package.json b/package.json index a0d101b..1332238 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "sharp": "^0.35.3" }, "dependencies": { + "axe-core": "^4.13.0", "imapflow": "^1.7.8", "mailparser": "^3.9.20", "nodemailer": "^10.0.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 490899b..59f24f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + axe-core: + specifier: ^4.13.0 + version: 4.13.0 imapflow: specifier: ^1.7.8 version: 1.7.8 @@ -532,6 +535,10 @@ packages: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} + axe-core@4.13.0: + resolution: {integrity: sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==} + engines: {node: '>=4'} + chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -1301,6 +1308,8 @@ snapshots: atomic-sleep@1.0.0: {} + axe-core@4.13.0: {} + chai@6.2.2: {} convert-source-map@2.0.0: {} diff --git a/src/registry.ts b/src/registry.ts index 9002fde..01dda5c 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -57,6 +57,7 @@ const SUMMARIES: Record = { torrent: 'Make a torrent out of a directory, and get it seeded', tts: 'Read text aloud and keep the audio', vid: 'Inspect, thumbnail, clip and shrink video, through ffmpeg', + wcag: 'Audit a site against WCAG with axe in headless Chrome, for the W3C report tool', }; /** The repository root, found from this file rather than from the cwd. */ diff --git a/src/wcag.ts b/src/wcag.ts new file mode 100644 index 0000000..1aa95e6 --- /dev/null +++ b/src/wcag.ts @@ -0,0 +1,1493 @@ +/** + * wcag — the automated half of a WCAG-EM evaluation, from the terminal. + * + * The W3C's WCAG-EM Report Tool (w3.org/WAI/eval/report-tool) is a web app + * with no command line and no headless mode: it is a form for the five steps + * of the Website Accessibility Conformance Evaluation Methodology, and the + * reason it cannot be a CLI is that most success criteria need a person to + * decide them. What a machine can do is the part that is machine-shaped: + * choose the structured sample (step 3), load every page in a real browser, + * run axe-core over it, and turn what axe found into assertions the tool + * already understands (step 4) — so an evaluator opens the tool with the + * sample and the automated failures filled in, and starts at the judgement + * calls rather than at an empty form. + * + * Three decisions shape this file: + * + * - **The browser is driven over CDP by hand.** Puppeteer and Playwright each + * bring a download of Chrome and a few megabytes of driver to do what this + * needs: open a page, wait for load, evaluate two scripts. Node 22 has a + * WebSocket, Chrome prints its DevTools URL on stderr, and the six protocol + * methods used here are stable. The box's own Chrome is found rather than + * fetched; the finder says where it looked when there is none. + * - **A pass is never asserted.** axe can prove a failure — an image with no + * alternative fails 1.1.1 wherever it is — but "no rule fired" proves + * nothing about the criterion as a whole, since its rules cover a part of + * each one. So a criterion with only passing checks lands in the tool as + * "cannot tell" with the checks listed, and "passed" is left for a person. + * The terminal summary shows the passes separately, because they are still + * useful to see. + * - **The evaluation file mirrors the tool's own export**, read from its + * source (src/stores/evaluationStore.js, src/data/jsonld/appContext.js) — + * the JSON-LD context, the `Evaluation` type, `defineScope` through + * `reportFindings`, a `Webpage` subject per sampled page whose id is its + * URL, and an `Assertion` per page and criterion with `test` set to the + * tool's own criterion ids (`WCAG22:non-text-content`). A file of that + * shape goes through the tool's "Open evaluation" rather than the beta + * assertion import, which is the path that restores the sample as well. + */ + +import { type ChildProcess, spawn } from 'node:child_process'; +import { accessSync, constants, mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { table } from './format.ts'; + +export class WcagError extends Error { + constructor(message: string) { + super(message); + this.name = 'WcagError'; + } +} + +// --------------------------------------------------------------------------- +// Success criteria +// --------------------------------------------------------------------------- + +export type Level = 'A' | 'AA' | 'AAA'; +export type WcagVersion = '2.1' | '2.2'; + +export const LEVELS: readonly Level[] = ['A', 'AA', 'AAA']; +export const WCAG_VERSIONS: readonly WcagVersion[] = ['2.1', '2.2']; +export const DEFAULT_LEVEL: Level = 'AA'; +export const DEFAULT_VERSION: WcagVersion = '2.2'; + +export interface Criterion { + /** `1.4.3` */ + num: string; + title: string; + level: Level; + /** + * The id the report tool uses for the criterion, `contrast-minimum`. These + * are the fragment ids of the WCAG 2.x specification, which is why they + * differ between versions for 2.5.5 and why 2.0's are not carried at all. + */ + id: string; + /** Which versions carry the criterion. 4.1.1 was removed in 2.2. */ + versions: readonly WcagVersion[]; + /** The 2.1 id, where it is not the 2.2 one. */ + id21?: string; +} + +const BOTH: readonly WcagVersion[] = ['2.1', '2.2']; +const ONLY_22: readonly WcagVersion[] = ['2.2']; +const ONLY_21: readonly WcagVersion[] = ['2.1']; + +const c = ( + num: string, + title: string, + level: Level, + id: string, + versions: readonly WcagVersion[] = BOTH, + id21?: string, +): Criterion => (id21 ? { num, title, level, id, versions, id21 } : { num, title, level, id, versions }); + +/** + * WCAG 2.1 and 2.2, as the report tool numbers and names them + * (src/data/wcag.json and src/locales/en/WCAG.json in its repository). + */ +export const CRITERIA: readonly Criterion[] = [ + c('1.1.1', 'Non-text Content', 'A', 'non-text-content'), + c('1.2.1', 'Audio-only and Video-only (Prerecorded)', 'A', 'audio-only-and-video-only-prerecorded'), + c('1.2.2', 'Captions (Prerecorded)', 'A', 'captions-prerecorded'), + c('1.2.3', 'Audio Description or Media Alternative (Prerecorded)', 'A', 'audio-description-or-media-alternative-prerecorded'), + c('1.2.4', 'Captions (Live)', 'AA', 'captions-live'), + c('1.2.5', 'Audio Description (Prerecorded)', 'AA', 'audio-description-prerecorded'), + c('1.2.6', 'Sign Language (Prerecorded)', 'AAA', 'sign-language-prerecorded'), + c('1.2.7', 'Extended Audio Description (Prerecorded)', 'AAA', 'extended-audio-description-prerecorded'), + c('1.2.8', 'Media Alternative (Prerecorded)', 'AAA', 'media-alternative-prerecorded'), + c('1.2.9', 'Audio-only (Live)', 'AAA', 'audio-only-live'), + c('1.3.1', 'Info and Relationships', 'A', 'info-and-relationships'), + c('1.3.2', 'Meaningful Sequence', 'A', 'meaningful-sequence'), + c('1.3.3', 'Sensory Characteristics', 'A', 'sensory-characteristics'), + c('1.3.4', 'Orientation', 'AA', 'orientation'), + c('1.3.5', 'Identify Input Purpose', 'AA', 'identify-input-purpose'), + c('1.3.6', 'Identify Purpose', 'AAA', 'identify-purpose'), + c('1.4.1', 'Use of Color', 'A', 'use-of-color'), + c('1.4.2', 'Audio Control', 'A', 'audio-control'), + c('1.4.3', 'Contrast (Minimum)', 'AA', 'contrast-minimum'), + c('1.4.4', 'Resize text', 'AA', 'resize-text'), + c('1.4.5', 'Images of Text', 'AA', 'images-of-text'), + c('1.4.6', 'Contrast (Enhanced)', 'AAA', 'contrast-enhanced'), + c('1.4.7', 'Low or No Background Audio', 'AAA', 'low-or-no-background-audio'), + c('1.4.8', 'Visual Presentation', 'AAA', 'visual-presentation'), + c('1.4.9', 'Images of Text (No Exception)', 'AAA', 'images-of-text-no-exception'), + c('1.4.10', 'Reflow', 'AA', 'reflow'), + c('1.4.11', 'Non-text Contrast', 'AA', 'non-text-contrast'), + c('1.4.12', 'Text Spacing', 'AA', 'text-spacing'), + c('1.4.13', 'Content on Hover or Focus', 'AA', 'content-on-hover-or-focus'), + c('2.1.1', 'Keyboard', 'A', 'keyboard'), + c('2.1.2', 'No Keyboard Trap', 'A', 'no-keyboard-trap'), + c('2.1.3', 'Keyboard (No Exception)', 'AAA', 'keyboard-no-exception'), + c('2.1.4', 'Character Key Shortcuts', 'A', 'character-key-shortcuts'), + c('2.2.1', 'Timing Adjustable', 'A', 'timing-adjustable'), + c('2.2.2', 'Pause, Stop, Hide', 'A', 'pause-stop-hide'), + c('2.2.3', 'No Timing', 'AAA', 'no-timing'), + c('2.2.4', 'Interruptions', 'AAA', 'interruptions'), + c('2.2.5', 'Re-authenticating', 'AAA', 're-authenticating'), + c('2.2.6', 'Timeouts', 'AAA', 'timeouts'), + c('2.3.1', 'Three Flashes or Below Threshold', 'A', 'three-flashes-or-below-threshold'), + c('2.3.2', 'Three Flashes', 'AAA', 'three-flashes'), + c('2.3.3', 'Animation from Interactions', 'AAA', 'animation-from-interactions'), + c('2.4.1', 'Bypass Blocks', 'A', 'bypass-blocks'), + c('2.4.2', 'Page Titled', 'A', 'page-titled'), + c('2.4.3', 'Focus Order', 'A', 'focus-order'), + c('2.4.4', 'Link Purpose (In Context)', 'A', 'link-purpose-in-context'), + c('2.4.5', 'Multiple Ways', 'AA', 'multiple-ways'), + c('2.4.6', 'Headings and Labels', 'AA', 'headings-and-labels'), + c('2.4.7', 'Focus Visible', 'AA', 'focus-visible'), + c('2.4.8', 'Location', 'AAA', 'location'), + c('2.4.9', 'Link Purpose (Link Only)', 'AAA', 'link-purpose-link-only'), + c('2.4.10', 'Section Headings', 'AAA', 'section-headings'), + c('2.4.11', 'Focus Not Obscured (Minimum)', 'AA', 'focus-not-obscured-minimum', ONLY_22), + c('2.4.12', 'Focus Not Obscured (Enhanced)', 'AAA', 'focus-not-obscured-enhanced', ONLY_22), + c('2.4.13', 'Focus Appearance', 'AAA', 'focus-appearance', ONLY_22), + c('2.5.1', 'Pointer Gestures', 'A', 'pointer-gestures'), + c('2.5.2', 'Pointer Cancellation', 'A', 'pointer-cancellation'), + c('2.5.3', 'Label in Name', 'A', 'label-in-name'), + c('2.5.4', 'Motion Actuation', 'A', 'motion-actuation'), + c('2.5.5', 'Target Size (Enhanced)', 'AAA', 'target-size-enhanced', BOTH, 'target-size'), + c('2.5.6', 'Concurrent Input Mechanisms', 'AAA', 'concurrent-input-mechanisms'), + c('2.5.7', 'Dragging Movements', 'AA', 'dragging-movements', ONLY_22), + c('2.5.8', 'Target Size (Minimum)', 'AA', 'target-size-minimum', ONLY_22), + c('3.1.1', 'Language of Page', 'A', 'language-of-page'), + c('3.1.2', 'Language of Parts', 'AA', 'language-of-parts'), + c('3.1.3', 'Unusual Words', 'AAA', 'unusual-words'), + c('3.1.4', 'Abbreviations', 'AAA', 'abbreviations'), + c('3.1.5', 'Reading Level', 'AAA', 'reading-level'), + c('3.1.6', 'Pronunciation', 'AAA', 'pronunciation'), + c('3.2.1', 'On Focus', 'A', 'on-focus'), + c('3.2.2', 'On Input', 'A', 'on-input'), + c('3.2.3', 'Consistent Navigation', 'AA', 'consistent-navigation'), + c('3.2.4', 'Consistent Identification', 'AA', 'consistent-identification'), + c('3.2.5', 'Change on Request', 'AAA', 'change-on-request'), + c('3.2.6', 'Consistent Help', 'A', 'consistent-help', ONLY_22), + c('3.3.1', 'Error Identification', 'A', 'error-identification'), + c('3.3.2', 'Labels or Instructions', 'A', 'labels-or-instructions'), + c('3.3.3', 'Error Suggestion', 'AA', 'error-suggestion'), + c('3.3.4', 'Error Prevention (Legal, Financial, Data)', 'AA', 'error-prevention-legal-financial-data'), + c('3.3.5', 'Help', 'AAA', 'help'), + c('3.3.6', 'Error Prevention (All)', 'AAA', 'error-prevention-all'), + c('3.3.7', 'Redundant Entry', 'A', 'redundant-entry', ONLY_22), + c('3.3.8', 'Accessible Authentication (Minimum)', 'AA', 'accessible-authentication-minimum', ONLY_22), + c('3.3.9', 'Accessible Authentication (Enhanced)', 'AAA', 'accessible-authentication-enhanced', ONLY_22), + c('4.1.1', 'Parsing', 'A', 'parsing', ONLY_21), + c('4.1.2', 'Name, Role, Value', 'A', 'name-role-value'), + c('4.1.3', 'Status Messages', 'AA', 'status-messages'), +]; + +const BY_NUM = new Map(CRITERIA.map((criterion) => [criterion.num, criterion])); + +export function criterion(num: string): Criterion | undefined { + return BY_NUM.get(num); +} + +/** The criteria a version carries, in specification order. */ +export function criteriaFor(version: WcagVersion): Criterion[] { + return CRITERIA.filter((criterion) => criterion.versions.includes(version)); +} + +/** The report tool's id for a criterion under a version: `WCAG22:contrast-minimum`. */ +export function criterionId(criterion: Criterion, version: WcagVersion): string { + const id = version === '2.1' && criterion.id21 ? criterion.id21 : criterion.id; + return `WCAG${version.replace('.', '')}:${id}`; +} + +const LEVEL_RANK: Record = { A: 1, AA: 2, AAA: 3 }; + +/** Is a criterion of `level` inside a conformance target? AA includes A. */ +export function withinTarget(level: Level, target: Level): boolean { + return LEVEL_RANK[level] <= LEVEL_RANK[target]; +} + +export function isLevel(value: string): value is Level { + return (LEVELS as readonly string[]).includes(value); +} + +export function isWcagVersion(value: string): value is WcagVersion { + return (WCAG_VERSIONS as readonly string[]).includes(value); +} + +/** `1.4.3` sorts before `1.4.10`, which a string sort gets wrong. */ +export function compareNums(a: string, b: string): number { + const left = a.split('.').map(Number); + const right = b.split('.').map(Number); + for (let index = 0; index < 3; index += 1) { + const difference = (left[index] ?? 0) - (right[index] ?? 0); + if (difference !== 0) return difference; + } + return 0; +} + +// --------------------------------------------------------------------------- +// axe tags +// --------------------------------------------------------------------------- + +/** + * The success criterion an axe tag names, or null. + * + * axe tags a rule `wcag143` for 1.4.3 and `wcag1412` for 1.4.12: principle, + * guideline, criterion, with no separators. The split is unambiguous because + * WCAG has four principles and no guideline past 5, so the first two digits + * are always one each and whatever follows is the criterion. + */ +export function criterionFromTag(tag: string): string | null { + const match = /^wcag([1-4])([1-5])(\d{1,2})$/.exec(tag); + if (!match) return null; + const num = `${match[1]}.${match[2]}.${Number(match[3])}`; + return BY_NUM.has(num) ? num : null; +} + +/** The criteria a rule's tags name, in specification order. */ +export function criteriaFromTags(tags: readonly string[]): string[] { + const nums = new Set(); + for (const tag of tags) { + const num = criterionFromTag(tag); + if (num) nums.add(num); + } + return [...nums].sort(compareNums); +} + +/** + * The axe tags that select the rules for a conformance target. + * + * axe files each rule under the version that introduced its criterion, so a + * 2.2 AA run wants the 2.0, 2.1 and 2.2 tags at A and AA. Best practices are + * left out on purpose: they are not success criteria and cannot fail one. + */ +export function axeTags(version: WcagVersion, level: Level): string[] { + const versions = version === '2.1' ? ['2', '21'] : ['2', '21', '22']; + const levels = LEVELS.filter((candidate) => withinTarget(candidate, level)); + const tags: string[] = []; + for (const prefix of versions) { + for (const suffix of levels) { + tags.push(`wcag${prefix}${suffix.toLowerCase()}`); + } + } + return tags; +} + +// --------------------------------------------------------------------------- +// The sample (WCAG-EM step 3) +// --------------------------------------------------------------------------- + +export type SampleMethod = 'auto' | 'sitemap' | 'links' | 'list'; +export const SAMPLE_METHODS: readonly SampleMethod[] = ['auto', 'sitemap', 'links', 'list']; + +export function isSampleMethod(value: string): value is SampleMethod { + return (SAMPLE_METHODS as readonly string[]).includes(value); +} + +const decodeXml = (value: string): string => + value + .replace(//g, '$1') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'|'/g, "'") + .trim(); + +export interface Sitemap { + /** Page URLs, from ``. */ + urls: string[]; + /** Child sitemaps, from `` in an index. */ + sitemaps: string[]; +} + +/** Read a sitemap or a sitemap index. Anything else yields nothing, not an error. */ +export function parseSitemap(xml: string): Sitemap { + const urls: string[] = []; + const sitemaps: string[] = []; + const entries = /<(url|sitemap)\b[^>]*>([\s\S]*?)<\/\1>/g; + let entry: RegExpExecArray | null; + while ((entry = entries.exec(xml)) !== null) { + const loc = /]*>([\s\S]*?)<\/loc>/.exec(entry[2] ?? ''); + if (!loc?.[1]) continue; + (entry[1] === 'sitemap' ? sitemaps : urls).push(decodeXml(loc[1])); + } + return { urls, sitemaps }; +} + +/** The sitemaps a robots.txt declares. */ +export function sitemapsFromRobots(robots: string): string[] { + return robots + .split(/\r?\n/) + .map((line) => /^\s*sitemap\s*:\s*(\S+)/i.exec(line)?.[1]) + .filter((url): url is string => Boolean(url)); +} + +/** Files that are not pages, by extension. A PDF has its own evaluation. */ +const NOT_A_PAGE = + /\.(pdf|jpe?g|png|gif|svg|webp|avif|ico|zip|gz|tar|mp3|mp4|webm|ogg|wav|css|js|mjs|json|xml|rss|atom|txt|woff2?|ttf|eot|csv|docx?|xlsx?|pptx?)$/i; + +/** + * The page's own URL with what makes two spellings the same page removed: + * the fragment, a default port, `index.html`, a trailing slash. + */ +export function normalizeUrl(value: string): string | null { + let url: URL; + try { + url = new URL(value); + } catch { + return null; + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') return null; + url.hash = ''; + url.pathname = url.pathname.replace(/\/index\.html?$/i, '/'); + if (url.pathname.length > 1) url.pathname = url.pathname.replace(/\/+$/, ''); + return url.href; +} + +/** + * Same-origin links on a page, absolute, deduplicated, in document order. + * + * A regular expression rather than a parser: the input is untrusted HTML and + * the question is only "which hrefs are here", which a DOM would answer with + * the same list at the cost of a dependency. + */ +export function sameOriginLinks(html: string, base: string): string[] { + const origin = new URL(base).origin; + const seen = new Set(); + const links: string[] = []; + const hrefs = /]*?\shref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi; + let match: RegExpExecArray | null; + while ((match = hrefs.exec(html)) !== null) { + const raw = decodeXml(match[1] ?? match[2] ?? match[3] ?? ''); + if (!raw || raw.startsWith('#')) continue; + let resolved: string | null; + try { + resolved = normalizeUrl(new URL(raw, base).href); + } catch { + continue; + } + if (!resolved || new URL(resolved).origin !== origin) continue; + if (NOT_A_PAGE.test(new URL(resolved).pathname)) continue; + if (seen.has(resolved)) continue; + seen.add(resolved); + links.push(resolved); + } + return links; +} + +/** + * Pick the structured sample. + * + * WCAG-EM wants pages that differ — the home page, one of each template, the + * forms, the odd one out — not the first N a crawler happened to list, which + * on most sites is N posts from one archive. So after the start page and any + * URLs named by hand, candidates are grouped by their first path segment and + * taken one group at a time, round-robin, until the limit. A blog with + * `/posts/…`, `/about`, `/pricing` and `/docs/…` yields one of each before a + * second post. + */ +export function chooseSample( + start: string, + candidates: readonly string[], + limit: number, + extra: readonly string[] = [], +): string[] { + const chosen: string[] = []; + const seen = new Set(); + const take = (url: string): boolean => { + const key = normalizeUrl(url); + if (!key || seen.has(key)) return false; + seen.add(key); + chosen.push(key); + return true; + }; + + take(start); + for (const url of extra) { + if (chosen.length >= limit) return chosen; + take(url); + } + + const groups = new Map(); + for (const url of candidates) { + const key = normalizeUrl(url); + if (!key || seen.has(key)) continue; + const segment = new URL(key).pathname.split('/')[1] ?? ''; + const group = groups.get(segment) ?? []; + group.push(key); + groups.set(segment, group); + } + + const queues = [...groups.values()]; + while (chosen.length < limit && queues.some((queue) => queue.length > 0)) { + for (const queue of queues) { + if (chosen.length >= limit) break; + const next = queue.shift(); + if (next !== undefined) take(next); + } + } + return chosen; +} + +export type FetchText = (url: string) => Promise; + +/** Fetch a text resource, or null on any failure: a missing sitemap is not an error. */ +export async function fetchText(url: string, timeoutMs = 15_000): Promise { + try { + const response = await fetch(url, { + headers: { 'user-agent': USER_AGENT, accept: 'text/html,application/xml,text/xml,text/plain;q=0.9,*/*;q=0.5' }, + signal: AbortSignal.timeout(timeoutMs), + redirect: 'follow', + }); + if (!response.ok) return null; + return await response.text(); + } catch { + return null; + } +} + +export interface SampleResult { + pages: string[]; + /** Where the candidates came from. */ + from: 'sitemap' | 'links' | 'list' | 'start'; + /** How many candidate pages the site offered before the limit. */ + candidates: number; +} + +const MAX_SITEMAPS = 8; +const MAX_CANDIDATES = 5000; + +/** + * Find the pages to audit. + * + * `sitemap` reads /sitemap.xml (and the ones robots.txt names), following an + * index into its first few children; `links` reads the start page's own + * same-origin links; `auto` tries the sitemap and falls back to links when it + * yields fewer pages than asked for; `list` audits only the start page and + * the URLs given by hand. + */ +export async function discoverSample( + start: string, + options: { pages: number; method: SampleMethod; extra?: readonly string[]; fetch?: FetchText }, +): Promise { + const { pages, method, extra = [], fetch: get = fetchText } = options; + const origin = new URL(start).origin; + + if (method === 'list') { + const chosen = chooseSample(start, [], pages, extra); + return { pages: chosen, from: extra.length > 0 ? 'list' : 'start', candidates: chosen.length }; + } + + let candidates: string[] = []; + let from: SampleResult['from'] = 'start'; + + if (method === 'sitemap' || method === 'auto') { + const roots = [`${origin}/sitemap.xml`]; + const robots = await get(`${origin}/robots.txt`); + if (robots) { + for (const url of sitemapsFromRobots(robots)) { + if (!roots.includes(url)) roots.push(url); + } + } + const queue = roots.slice(0, MAX_SITEMAPS); + const visited = new Set(); + while (queue.length > 0 && candidates.length < MAX_CANDIDATES && visited.size < MAX_SITEMAPS) { + const url = queue.shift(); + if (url === undefined || visited.has(url)) continue; + visited.add(url); + const xml = await get(url); + if (!xml) continue; + const sitemap = parseSitemap(xml); + candidates.push(...sitemap.urls.filter((page) => page.startsWith(origin) && !NOT_A_PAGE.test(page))); + queue.push(...sitemap.sitemaps); + } + if (candidates.length > 0) from = 'sitemap'; + } + + if (method === 'links' || (method === 'auto' && candidates.length + 1 + extra.length < pages)) { + const html = await get(start); + if (html) { + const links = sameOriginLinks(html, start); + if (links.length > 0) { + candidates = candidates.concat(links); + if (from === 'start') from = 'links'; + } + } + } + + return { pages: chooseSample(start, candidates, pages, extra), from, candidates: candidates.length }; +} + +// --------------------------------------------------------------------------- +// The browser +// --------------------------------------------------------------------------- + +export const USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) cli-tools/wcag (+https://github.com/profullstack/cli-tools)'; + +/** Where the box keeps a Chrome that Chrome's own libraries are not installed for. */ +const CHROME_DEPS = join(homedir(), '.local', 'share', 'chrome-deps'); + +const isExecutable = (path: string): boolean => { + try { + accessSync(path, constants.X_OK); + return true; + } catch { + return false; + } +}; + +const listDir = (path: string): string[] => { + try { + return readdirSync(path); + } catch { + return []; + } +}; + +/** `linux-152.0.7977.42` after `linux-131.0.6778.204`: newest first by version, not by string. */ +const byVersionDesc = (a: string, b: string): number => { + const parse = (name: string): number[] => (/(\d+(?:\.\d+)*)/.exec(name)?.[1] ?? '0').split('.').map(Number); + const left = parse(a); + const right = parse(b); + for (let index = 0; index < Math.max(left.length, right.length); index += 1) { + const difference = (right[index] ?? 0) - (left[index] ?? 0); + if (difference !== 0) return difference; + } + return 0; +}; + +/** + * Every place a Chrome might be, in the order they are tried. + * + * `CHROME_PATH` first, because it is the one the caller chose. Then the names + * a package manager installs, then the caches Puppeteer and Playwright keep + * (newest build first), then the macOS bundle. A headless shell is a real + * Chrome for this purpose — it is what axe needs, a DOM and a renderer. + */ +export function chromeCandidates(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): string[] { + const candidates: string[] = []; + if (env.CHROME_PATH) candidates.push(env.CHROME_PATH); + + for (const name of ['google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser', 'chrome', 'chrome-headless-shell']) { + for (const dir of (env.PATH ?? '').split(':').filter(Boolean)) { + candidates.push(join(dir, name)); + } + } + candidates.push('/opt/google/chrome/chrome'); + + const puppeteer = join(home, '.cache', 'puppeteer'); + for (const flavour of ['chrome', 'chrome-headless-shell']) { + for (const build of listDir(join(puppeteer, flavour)).sort(byVersionDesc)) { + const dir = join(puppeteer, flavour, build); + const inner = listDir(dir).find((entry) => entry.startsWith(flavour)); + if (inner) candidates.push(join(dir, inner, flavour)); + } + } + + const playwright = join(home, '.cache', 'ms-playwright'); + for (const build of listDir(playwright).sort(byVersionDesc)) { + if (build.startsWith('chromium_headless_shell-')) candidates.push(join(playwright, build, 'chrome-linux', 'headless_shell')); + else if (build.startsWith('chromium-')) candidates.push(join(playwright, build, 'chrome-linux', 'chrome')); + } + + candidates.push('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'); + candidates.push('/Applications/Chromium.app/Contents/MacOS/Chromium'); + return candidates; +} + +export function findChrome( + env: NodeJS.ProcessEnv = process.env, + home: string = homedir(), + executable: (path: string) => boolean = isExecutable, +): string | null { + return chromeCandidates(env, home).find(executable) ?? null; +} + +/** + * The environment Chrome runs with. + * + * A Puppeteer or Playwright build is a bare binary: it expects the distro's + * GTK, ATK and X libraries and dies on `libatk-1.0.so.0` without them. On a + * box with no root that cannot install them, they are staged under + * `~/.local/share/chrome-deps` instead, and pointing `LD_LIBRARY_PATH` and + * `FONTCONFIG_FILE` there is what makes the binary run. Done here rather than + * in a wrapper so the command works the same whichever Chrome was found. + */ +export function browserEnv(env: NodeJS.ProcessEnv = process.env, deps: string = CHROME_DEPS): NodeJS.ProcessEnv { + const result: NodeJS.ProcessEnv = { ...env }; + const lib = join(deps, 'usr', 'lib', 'x86_64-linux-gnu'); + if (listDir(lib).length > 0 && !(env.LD_LIBRARY_PATH ?? '').split(':').includes(lib)) { + result.LD_LIBRARY_PATH = env.LD_LIBRARY_PATH ? `${lib}:${env.LD_LIBRARY_PATH}` : lib; + } + const fonts = join(deps, 'etc', 'fonts', 'fonts.conf'); + if (!env.FONTCONFIG_FILE && isExecutable(join(deps, 'etc', 'fonts')) ) { + result.FONTCONFIG_FILE = fonts; + } + return result; +} + +export const NO_CHROME = `no Chrome found. Set CHROME_PATH to a Chrome or Chromium binary, install one +(apt install chromium, brew install --cask google-chrome), or let Puppeteer +or Playwright fetch one (npx puppeteer browsers install chrome).`; + +interface CdpMessage { + id?: number; + method?: string; + params?: Record; + sessionId?: string; + result?: Record; + error?: { message: string }; +} + +/** + * The slice of the Chrome DevTools Protocol this needs: a request-response + * channel with ids, and events by name. Flat session mode, so one socket + * serves the browser and every page. + */ +export class Cdp { + private nextId = 1; + private readonly pending = new Map) => void; reject: (error: Error) => void }>(); + private readonly listeners = new Set<(message: CdpMessage) => void>(); + private readonly socket: WebSocket; + + constructor(socket: WebSocket) { + this.socket = socket; + socket.addEventListener('message', (event) => { + const message = JSON.parse(String(event.data)) as CdpMessage; + if (message.id !== undefined && this.pending.has(message.id)) { + const { resolve, reject } = this.pending.get(message.id)!; + this.pending.delete(message.id); + if (message.error) reject(new WcagError(message.error.message)); + else resolve(message.result ?? {}); + return; + } + for (const listener of this.listeners) listener(message); + }); + socket.addEventListener('close', () => { + for (const { reject } of this.pending.values()) reject(new WcagError('browser closed')); + this.pending.clear(); + }); + } + + static async connect(url: string): Promise { + const socket = new WebSocket(url); + await new Promise((resolve, reject) => { + socket.addEventListener('open', () => resolve(), { once: true }); + socket.addEventListener('error', () => reject(new WcagError(`could not connect to ${url}`)), { once: true }); + }); + return new Cdp(socket); + } + + send(method: string, params: Record = {}, sessionId?: string): Promise> { + const id = this.nextId; + this.nextId += 1; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.socket.send(JSON.stringify(sessionId ? { id, method, params, sessionId } : { id, method, params })); + }); + } + + /** Resolve on the next event of `method` for a session, or reject after `timeoutMs`. */ + waitFor(method: string, sessionId: string, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.listeners.delete(listener); + reject(new WcagError(`timed out after ${Math.round(timeoutMs / 1000)}s waiting for the page to load`)); + }, timeoutMs); + const listener = (message: CdpMessage): void => { + if (message.method === method && message.sessionId === sessionId) { + clearTimeout(timer); + this.listeners.delete(listener); + resolve(); + } + }; + this.listeners.add(listener); + }); + } + + close(): void { + this.socket.close(); + } +} + +export interface Browser { + cdp: Cdp; + path: string; + close(): Promise; +} + +export interface LaunchOptions { + chrome?: string; + env?: NodeJS.ProcessEnv; + /** Chrome's sandbox needs user namespaces; a container or root often lacks them. */ + sandbox?: boolean; + timeoutMs?: number; +} + +/** Start a headless Chrome and connect to it. */ +export async function launchBrowser(options: LaunchOptions = {}): Promise { + const env = options.env ?? process.env; + const path = options.chrome ?? findChrome(env); + if (!path) throw new WcagError(NO_CHROME); + if (!isExecutable(path)) throw new WcagError(`${path} is not an executable`); + + const profile = mkdtempSync(join(tmpdir(), 'wcag-chrome-')); + const sandbox = options.sandbox ?? !(env.CHROME_NO_SANDBOX || process.getuid?.() === 0); + const args = [ + '--headless=new', + '--remote-debugging-port=0', + `--user-data-dir=${profile}`, + '--no-first-run', + '--no-default-browser-check', + '--disable-gpu', + '--disable-extensions', + '--disable-background-networking', + '--hide-scrollbars', + '--window-size=1280,800', + ...(sandbox ? [] : ['--no-sandbox']), + 'about:blank', + ]; + + const child: ChildProcess = spawn(path, args, { env: browserEnv(env), stdio: ['ignore', 'ignore', 'pipe'] }); + const cleanup = (): void => { + try { + rmSync(profile, { recursive: true, force: true }); + } catch { + // A profile Chrome is still holding open is removed on the next run's tmpdir sweep. + } + }; + + const url = await new Promise((resolve, reject) => { + let stderr = ''; + const timer = setTimeout(() => reject(new WcagError(`${path} did not start within ${(options.timeoutMs ?? 20_000) / 1000}s\n${stderr}`)), options.timeoutMs ?? 20_000); + child.stderr?.on('data', (chunk: Buffer) => { + stderr += chunk.toString(); + const match = /DevTools listening on (ws:\/\/\S+)/.exec(stderr); + if (match?.[1]) { + clearTimeout(timer); + resolve(match[1]); + } + }); + child.on('exit', (code) => { + clearTimeout(timer); + reject(new WcagError(`${path} exited with ${code ?? 'a signal'} before it was ready\n${stderr.trim()}`)); + }); + child.on('error', (error) => { + clearTimeout(timer); + reject(new WcagError(`${path}: ${error.message}`)); + }); + }).catch((error: Error) => { + child.kill(); + cleanup(); + throw error; + }); + + const cdp = await Cdp.connect(url); + return { + cdp, + path, + async close() { + try { + await Promise.race([cdp.send('Browser.close'), new Promise((resolve) => setTimeout(resolve, 2000))]); + } catch { + // Already gone. + } + cdp.close(); + child.kill(); + cleanup(); + }, + }; +} + +// --------------------------------------------------------------------------- +// The audit (WCAG-EM step 4, the automated part) +// --------------------------------------------------------------------------- + +const require = createRequire(import.meta.url); + +/** axe-core's browser bundle, read from the installed package rather than fetched. */ +export function axeSource(): string { + return readFileSync(require.resolve('axe-core/axe.min.js'), 'utf8'); +} + +export function axeVersion(): string { + return (require('axe-core/package.json') as { version: string }).version; +} + +export interface RuleFinding { + rule: string; + impact: string | null; + help: string; + helpUrl: string; + /** Success criteria the rule is filed under, `1.4.3`. */ + criteria: string[]; + /** How many elements it applied to. */ + nodes: number; + /** CSS selectors of the first few of them. */ + targets: string[]; +} + +export interface PageResult { + url: string; + /** Where the browser ended up after redirects. */ + finalUrl: string; + title: string; + ok: boolean; + error?: string; + violations: RuleFinding[]; + incomplete: RuleFinding[]; + passes: RuleFinding[]; + /** Rules that found nothing to check on the page. */ + inapplicable: string[]; + /** Milliseconds from navigation to axe finishing. */ + ms: number; +} + +export interface Report { + tool: 'cli-tools wcag'; + axeVersion: string; + wcagVersion: WcagVersion; + level: Level; + /** The origin the sample was drawn from. */ + site: string; + startedAt: string; + finishedAt: string; + sample: { from: SampleResult['from']; candidates: number }; + pages: PageResult[]; +} + +interface AxeNode { + target: unknown[]; +} + +interface AxeResult { + id: string; + impact: string | null; + help: string; + helpUrl: string; + tags: string[]; + nodes: AxeNode[]; +} + +interface AxeRun { + url: string; + violations: AxeResult[]; + incomplete: AxeResult[]; + passes: AxeResult[]; + inapplicable: AxeResult[]; +} + +const MAX_TARGETS = 3; + +const toFinding = (result: AxeResult): RuleFinding => ({ + rule: result.id, + impact: result.impact ?? null, + help: result.help, + helpUrl: result.helpUrl, + criteria: criteriaFromTags(result.tags), + nodes: result.nodes.length, + targets: result.nodes.slice(0, MAX_TARGETS).map((node) => node.target.map(String).join(' ')), +}); + +/** The in-page half: run axe over the loaded document and hand back what it found. */ +const RUN_AXE = (tags: string[]): string => `axe.run(document, { + runOnly: { type: 'tag', values: ${JSON.stringify(tags)} }, + resultTypes: ['violations', 'incomplete', 'passes', 'inapplicable'], +}).then((results) => JSON.stringify({ + url: results.url, + violations: results.violations, + incomplete: results.incomplete, + passes: results.passes, + inapplicable: results.inapplicable.map((result) => ({ id: result.id, impact: null, help: '', helpUrl: '', tags: [], nodes: [] })), +}))`; + +export interface AuditOptions { + version: WcagVersion; + level: Level; + timeoutMs?: number; + /** Milliseconds to let the page settle after load before running axe. */ + settleMs?: number; + axe?: string; +} + +/** Load one page in the browser and run axe over it. Never throws: a page that will not load is a result too. */ +export async function auditPage(browser: Browser, url: string, options: AuditOptions): Promise { + const { cdp } = browser; + const timeoutMs = options.timeoutMs ?? 30_000; + const started = Date.now(); + const failure = (error: string): PageResult => ({ + url, + finalUrl: url, + title: '', + ok: false, + error, + violations: [], + incomplete: [], + passes: [], + inapplicable: [], + ms: Date.now() - started, + }); + + let targetId: string | undefined; + try { + ({ targetId } = (await cdp.send('Target.createTarget', { url: 'about:blank' })) as { targetId: string }); + const { sessionId } = (await cdp.send('Target.attachToTarget', { targetId, flatten: true })) as { sessionId: string }; + + await cdp.send('Page.enable', {}, sessionId); + await cdp.send('Network.setUserAgentOverride', { userAgent: USER_AGENT }, sessionId); + await cdp.send('Emulation.setDeviceMetricsOverride', { width: 1280, height: 800, deviceScaleFactor: 1, mobile: false }, sessionId); + + const loaded = cdp.waitFor('Page.loadEventFired', sessionId, timeoutMs); + const navigation = (await cdp.send('Page.navigate', { url }, sessionId)) as { errorText?: string }; + if (navigation.errorText) { + loaded.catch(() => undefined); + return failure(navigation.errorText); + } + await loaded; + await new Promise((resolve) => setTimeout(resolve, options.settleMs ?? 500)); + + const evaluate = async (expression: string, awaitPromise = false): Promise => { + const { result, exceptionDetails } = (await cdp.send( + 'Runtime.evaluate', + { expression, awaitPromise, returnByValue: true }, + sessionId, + )) as { result: { value?: unknown }; exceptionDetails?: { text: string; exception?: { description?: string } } }; + if (exceptionDetails) { + throw new WcagError(exceptionDetails.exception?.description ?? exceptionDetails.text); + } + return result.value; + }; + + const page = (await evaluate('JSON.stringify({ title: document.title, href: location.href })')) as string; + const { title, href } = JSON.parse(page) as { title: string; href: string }; + + await evaluate(options.axe ?? axeSource()); + const raw = (await evaluate(RUN_AXE(axeTags(options.version, options.level)), true)) as string; + const run = JSON.parse(raw) as AxeRun; + + return { + url, + finalUrl: href || run.url || url, + title, + ok: true, + violations: run.violations.map(toFinding), + incomplete: run.incomplete.map(toFinding), + passes: run.passes.map(toFinding), + inapplicable: run.inapplicable.map((result) => result.id), + ms: Date.now() - started, + }; + } catch (error) { + return failure((error as Error).message); + } finally { + if (targetId) { + await cdp.send('Target.closeTarget', { targetId }).catch(() => undefined); + } + } +} + +export interface RunOptions extends AuditOptions { + onPage?: (result: PageResult, index: number, total: number) => void; +} + +/** Audit every page of a sample in one browser, in order. */ +export async function audit(browser: Browser, sample: SampleResult, options: RunOptions): Promise { + const startedAt = new Date().toISOString(); + const axe = options.axe ?? axeSource(); + const pages: PageResult[] = []; + for (const [index, url] of sample.pages.entries()) { + const result = await auditPage(browser, url, { ...options, axe }); + pages.push(result); + options.onPage?.(result, index, sample.pages.length); + } + return { + tool: 'cli-tools wcag', + axeVersion: axeVersion(), + wcagVersion: options.version, + level: options.level, + site: new URL(sample.pages[0] ?? 'https://invalid').origin, + startedAt, + finishedAt: new Date().toISOString(), + sample: { from: sample.from, candidates: sample.candidates }, + pages, + }; +} + +// --------------------------------------------------------------------------- +// Reading a report +// --------------------------------------------------------------------------- + +export type Outcome = 'failed' | 'cantTell' | 'passed' | 'untested'; + +export interface CriterionOutcome { + outcome: Outcome; + violations: RuleFinding[]; + incomplete: RuleFinding[]; + passes: RuleFinding[]; +} + +/** + * What a page's axe results say about each criterion. + * + * A violation fails the criterion. Anything short of that is "cannot tell": + * an incomplete check needs a person, and so does a criterion axe only + * checked part of — which is every criterion, so passing checks are recorded + * but never promoted to a pass. A criterion no rule touched is untested. + */ +export function pageOutcomes(page: PageResult): Map { + const outcomes = new Map(); + const entry = (num: string): CriterionOutcome => { + let existing = outcomes.get(num); + if (!existing) { + existing = { outcome: 'untested', violations: [], incomplete: [], passes: [] }; + outcomes.set(num, existing); + } + return existing; + }; + for (const finding of page.violations) for (const num of finding.criteria) entry(num).violations.push(finding); + for (const finding of page.incomplete) for (const num of finding.criteria) entry(num).incomplete.push(finding); + for (const finding of page.passes) for (const num of finding.criteria) entry(num).passes.push(finding); + for (const value of outcomes.values()) { + value.outcome = value.violations.length > 0 ? 'failed' : 'cantTell'; + } + return outcomes; +} + +export interface CriterionSummary { + num: string; + title: string; + level: Level; + /** Pages with a violation. */ + failing: number; + /** Pages with an incomplete check and no violation. */ + review: number; + /** Pages where every check passed. */ + passing: number; + /** Elements in violation across the sample. */ + nodes: number; + /** The rule with the most elements in violation. */ + worst: { rule: string; nodes: number; impact: string | null } | null; +} + +/** One row per criterion that any rule touched, in specification order. */ +export function summarize(report: Report): CriterionSummary[] { + const rows = new Map }>(); + for (const page of report.pages) { + if (!page.ok) continue; + for (const [num, outcome] of pageOutcomes(page)) { + const known = criterion(num); + if (!known) continue; + let row = rows.get(num); + if (!row) { + row = { num, title: known.title, level: known.level, failing: 0, review: 0, passing: 0, nodes: 0, worst: null, byRule: new Map() }; + rows.set(num, row); + } + if (outcome.violations.length > 0) { + row.failing += 1; + for (const finding of outcome.violations) { + row.nodes += finding.nodes; + const rule = row.byRule.get(finding.rule) ?? { nodes: 0, impact: finding.impact }; + rule.nodes += finding.nodes; + row.byRule.set(finding.rule, rule); + } + } else if (outcome.incomplete.length > 0) { + row.review += 1; + } else { + row.passing += 1; + } + } + } + return [...rows.values()] + .sort((a, b) => compareNums(a.num, b.num)) + .map(({ byRule, ...row }) => { + const worst = [...byRule.entries()].sort((a, b) => b[1].nodes - a[1].nodes)[0]; + return { ...row, worst: worst ? { rule: worst[0], nodes: worst[1].nodes, impact: worst[1].impact } : null }; + }); +} + +export interface Totals { + pages: number; + loaded: number; + failingPages: number; + /** Criteria within the target with at least one violation. */ + failingCriteria: number; + /** Criteria within the target needing review and not failing. */ + reviewCriteria: number; + nodes: number; +} + +export function totals(report: Report, rows: CriterionSummary[] = summarize(report)): Totals { + const inTarget = rows.filter((row) => withinTarget(row.level, report.level)); + return { + pages: report.pages.length, + loaded: report.pages.filter((page) => page.ok).length, + failingPages: report.pages.filter((page) => page.ok && page.violations.some((finding) => finding.criteria.length > 0)).length, + failingCriteria: inTarget.filter((row) => row.failing > 0).length, + reviewCriteria: inTarget.filter((row) => row.failing === 0 && row.review > 0).length, + nodes: inTarget.reduce((sum, row) => sum + row.nodes, 0), + }; +} + +/** Did anything within the conformance target fail? The exit code. */ +export function hasFailures(report: Report): boolean { + return totals(report).failingCriteria > 0; +} + +const plural = (count: number, noun: string): string => `${count} ${noun}${count === 1 ? '' : 's'}`; + +/** The terminal summary: a table of criteria, then one line of totals. */ +export function formatSummary(report: Report, rows: CriterionSummary[] = summarize(report)): string { + const lines: string[] = []; + const header = ['SC', 'Level', 'Criterion', 'Fail', 'Review', 'Pass', 'Worst rule']; + const body = rows.map((row) => [ + row.num, + row.level, + row.title, + row.failing > 0 ? String(row.failing) : '-', + row.review > 0 ? String(row.review) : '-', + row.passing > 0 ? String(row.passing) : '-', + row.worst ? `${row.worst.rule} (${plural(row.worst.nodes, 'element')}${row.worst.impact ? `, ${row.worst.impact}` : ''})` : '', + ]); + if (body.length > 0) { + lines.push(table([header, ...body])); + lines.push(''); + } + + const sum = totals(report, rows); + const broken = report.pages.filter((page) => !page.ok); + lines.push( + `WCAG ${report.wcagVersion} level ${report.level}: ${plural(sum.failingCriteria, 'criterion').replace('criterions', 'criteria')} failing on ${plural(sum.failingPages, 'page')} of ${sum.loaded}` + + ` (${plural(sum.nodes, 'element')}), ${sum.reviewCriteria} to review by hand. axe-core ${report.axeVersion}.`, + ); + for (const page of broken) { + lines.push(` could not load ${page.url}: ${page.error ?? 'unknown error'}`); + } + return lines.join('\n'); +} + +/** A Markdown version of the same, with the failing rules under each criterion. */ +export function toMarkdown(report: Report, rows: CriterionSummary[] = summarize(report)): string { + const sum = totals(report, rows); + const lines: string[] = []; + lines.push(`# Accessibility audit of ${report.site}`); + lines.push(''); + lines.push(`WCAG ${report.wcagVersion} level ${report.level}, automated checks only (axe-core ${report.axeVersion}), ${report.startedAt.slice(0, 10)}.`); + lines.push(''); + lines.push(`${sum.failingCriteria} criteria failing on ${sum.failingPages} of ${sum.loaded} pages; ${sum.reviewCriteria} more need a person. A criterion with no failure here is not passed: automated checks cover a part of each one.`); + lines.push(''); + lines.push('## Sample'); + lines.push(''); + for (const page of report.pages) { + lines.push(`- ${page.ok ? `[${page.title || page.url}](${page.finalUrl})` : `${page.url} — could not load (${page.error ?? 'unknown'})`}`); + } + lines.push(''); + lines.push('## Findings'); + lines.push(''); + if (rows.every((row) => row.failing === 0)) { + lines.push('No automated check failed.'); + lines.push(''); + } + for (const row of rows) { + if (row.failing === 0) continue; + lines.push(`### ${row.num} ${row.title} (${row.level}) — fails on ${plural(row.failing, 'page')}`); + lines.push(''); + const rules = new Map(); + for (const page of report.pages) { + for (const finding of page.violations) { + if (!finding.criteria.includes(row.num)) continue; + const rule = rules.get(finding.rule) ?? { finding, pages: [] }; + rule.pages.push(page.finalUrl); + rules.set(finding.rule, rule); + } + } + for (const { finding, pages } of rules.values()) { + lines.push(`- **${finding.rule}**${finding.impact ? ` (${finding.impact})` : ''}: ${finding.help}. [How to fix](${finding.helpUrl}). ${plural(pages.length, 'page')}${finding.targets.length > 0 ? `, e.g. \`${finding.targets[0]}\`` : ''}.`); + } + lines.push(''); + } + const review = rows.filter((row) => row.failing === 0 && row.review > 0); + if (review.length > 0) { + lines.push('## Needs a person'); + lines.push(''); + for (const row of review) lines.push(`- ${row.num} ${row.title} (${row.level}), ${plural(row.review, 'page')}`); + lines.push(''); + } + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// The report tool's evaluation file +// --------------------------------------------------------------------------- + +/** The version of the report tool whose export this mirrors. */ +export const REPORT_TOOL_VERSION = '4.1.0'; +export const REPORT_TOOL_URL = 'https://www.w3.org/WAI/eval/report-tool/'; + +/** + * The JSON-LD context the report tool writes and reads + * (src/data/jsonld/appContext.js, `exportContext`), verbatim. + */ +export const EVALUATION_CONTEXT = { + reporter: 'http://github.com/w3c/wai-wcag-em-report-tool/', + wcagem: 'http://www.w3.org/TR/WCAG-EM/#', + Evaluation: 'wcagem:procedure', + defineScope: 'wcagem:step1', + scope: 'wcagem:step1a', + step1b: { '@id': 'wcagem:step1b', '@type': '@id' }, + conformanceTarget: 'step1b', + accessibilitySupportBaseline: 'wcagem:step1c', + additionalEvaluationRequirements: 'wcagem:step1d', + exploreTarget: 'wcagem:step2', + essentialFunctionality: 'wcagem:step2b', + pageTypeVariety: 'wcagem:step2c', + technologiesReliedUpon: 'wcagem:step2d', + selectSample: 'wcagem:step3', + structuredSample: 'wcagem:step3a', + randomSample: 'wcagem:step3b', + Website: 'wcagem:website', + Webpage: 'wcagem:webpage', + auditSample: 'wcagem:step4', + reportFindings: 'wcagem:step5', + documentSteps: 'wcagem:step5a', + commissioner: 'wcagem:commissioner', + evaluator: 'wcagem:evaluator', + evaluationSpecifics: 'wcagem:step5b', + WCAG: 'http://www.w3.org/TR/WCAG/#', + WCAG20: 'http://www.w3.org/TR/WCAG20/#', + WCAG21: 'http://www.w3.org/TR/WCAG21/#', + WAI: 'http://www.w3.org/WAI/', + A: 'WAI:WCAG2A-Conformance', + AA: 'WAI:WCAG2AA-Conformance', + AAA: 'WAI:WCAG2AAA-Conformance', + wcagVersion: 'WAI:standards-guidelines/wcag/#versions', + reportToolVersion: 'wcagem:reportToolVersion', + earl: 'http://www.w3.org/ns/earl#', + Assertion: 'earl:Assertion', + TestMode: 'earl:TestMode', + TestCriterion: 'earl:TestCriterion', + TestCase: 'earl:TestCase', + TestRequirement: 'earl:TestRequirement', + TestSubject: 'earl:TestSubject', + TestResult: 'earl:TestResult', + OutcomeValue: 'earl:OutcomeValue', + Pass: 'earl:Pass', + Fail: 'earl:Fail', + CannotTell: 'earl:CannotTell', + NotApplicable: 'earl:NotApplicable', + NotTested: 'earl:NotTested', + assertedBy: 'earl:assertedBy', + mode: 'earl:mode', + result: 'earl:result', + subject: 'earl:subject', + test: 'earl:test', + outcome: 'earl:outcome', + dcterms: 'http://purl.org/dc/terms/', + title: 'dcterms:title', + description: 'dcterms:description', + summary: 'dcterms:summary', + date: 'dcterms:date', + hasPart: 'dcterms:hasPart', + isPartOf: 'dcterms:isPartOf', + id: '@id', + type: '@type', + language: '@language', +} as const; + +/* + * No `WCAG22` prefix is added to the context, on purpose. The tool's own + * context stops at WCAG21, so a 2.2 criterion id like `WCAG22:reflow` reaches + * its JSON-LD processor as an IRI with an unknown scheme, which expansion and + * compaction both leave untouched, and its importer then reads the id off the + * last colon. Defining the prefix here would expand the id to the full + * `http://www.w3.org/TR/WCAG22/#reflow`, which the tool's context cannot fold + * back, and every 2.2 assertion would be dropped on open. Found by replaying + * the tool's open() with its own jsonld version, not by reading the code. + */ + +const OUTCOME_TYPES: Record = { + passed: 'Pass', + failed: 'Fail', + cantTell: 'CannotTell', + untested: 'NotTested', +}; + +export interface EvaluationOptions { + /** The site's name, for the scope. Defaults to its host. */ + site?: string; + title?: string; + evaluator?: string; + commissioner?: string; +} + +/** + * A page name the tool can tell apart from the others'. + * + * The tool finds a sampled page by its title, so two pages titled "Blog" would + * collapse into one; the path is appended to the second. A page with no title + * is named by its path, which is what a person would call it anyway. + */ +export function pageTitles(pages: readonly PageResult[]): string[] { + const used = new Map(); + return pages.map((page) => { + const path = new URL(page.finalUrl).pathname; + const base = page.title.trim() || path; + const count = used.get(base) ?? 0; + used.set(base, count + 1); + return count === 0 ? base : `${base} (${path})`; + }); +} + +const describe = (outcome: CriterionOutcome, page: PageResult, report: Report): string => { + const lines: string[] = []; + const list = (label: string, findings: RuleFinding[]): void => { + if (findings.length === 0) return; + lines.push(`${label} (axe-core ${report.axeVersion}, ${page.finalUrl}):`); + for (const finding of findings) { + const where = finding.targets.length > 0 ? ` e.g. ${finding.targets.join(' | ')}` : ''; + lines.push(`- ${finding.rule}${finding.impact ? ` [${finding.impact}]` : ''}: ${finding.help} — ${plural(finding.nodes, 'element')}.${where} ${finding.helpUrl}`); + } + }; + list('Automated failures', outcome.violations); + list('Checks that need a person', outcome.incomplete); + if (outcome.passes.length > 0) { + lines.push(`Passing checks: ${outcome.passes.map((finding) => finding.rule).join(', ')}. Automated checks cover a part of this criterion; the rest is manual.`); + } + return lines.join('\n'); +}; + +/** + * The report tool's own evaluation file, with the sample and the automated + * results filled in. Open it in the tool with "Open evaluation". + */ +export function toEvaluation(report: Report, options: EvaluationOptions = {}): Record { + const { wcagVersion: version } = report; + const date = report.finishedAt; + const host = new URL(report.site).host; + const site = options.site ?? host; + const loaded = report.pages.filter((page) => page.ok); + const titles = pageTitles(loaded); + + const website = { + id: '_:subject_0', + type: ['TestSubject', 'Website'], + date, + title: site, + description: report.site, + }; + + const subjects = loaded.map((page, index) => ({ + id: page.finalUrl, + type: ['TestSubject', 'Webpage'], + date, + title: titles[index] ?? page.finalUrl, + description: page.finalUrl, + })); + + const assertor = { + id: 'https://github.com/profullstack/cli-tools#wcag', + title: `cli-tools wcag (axe-core ${report.axeVersion})`, + }; + + const assertions: Record[] = []; + loaded.forEach((page, index) => { + const subject = subjects[index]!; + for (const [num, outcome] of pageOutcomes(page)) { + const known = criterion(num); + if (!known || !known.versions.includes(version)) continue; + assertions.push({ + type: ['Assertion'], + date, + assertedBy: assertor, + mode: 'earl:automatic', + subject: { id: subject.id, type: subject.type, title: subject.title, description: subject.description }, + test: { + id: criterionId(known, version), + type: ['TestCriterion', 'TestRequirement'], + title: `${num} ${known.title}`, + num, + }, + result: { + type: ['TestResult'], + date, + outcome: { id: `earl:${outcome.outcome}`, type: ['OutcomeValue', OUTCOME_TYPES[outcome.outcome]] }, + description: describe(outcome, page, report), + }, + }); + } + }); + + const sum = totals(report); + const summary = + `Automated checks by cli-tools wcag (axe-core ${report.axeVersion}) on ${report.startedAt.slice(0, 10)}: ` + + `${sum.failingCriteria} criteria failing on ${sum.failingPages} of ${sum.loaded} sampled pages. ` + + 'Every criterion still needs a person: an automated failure is a failure, an automated pass is not a pass.'; + + return { + '@context': EVALUATION_CONTEXT, + '@type': 'Evaluation', + '@language': 'en', + reportToolVersion: REPORT_TOOL_VERSION, + defineScope: { + '@id': '_:defineScope', + scope: website, + wcagVersion: version, + conformanceTarget: report.level, + accessibilitySupportBaseline: '', + additionalEvaluationRequirements: '', + }, + exploreTarget: { + '@id': '_:exploreTarget', + technologiesReliedUpon: [], + essentialFunctionality: '', + pageTypeVariety: '', + }, + selectSample: { + '@id': '_:selectSample', + randomSample: [], + structuredSample: subjects, + }, + auditSample: assertions, + reportFindings: { + documentSteps: [{ '@id': '_:about' }, { '@id': '_:defineScope' }, { '@id': '_:exploreTarget' }, { '@id': '_:selectSample' }], + commissioner: options.commissioner ?? '', + date, + evaluator: options.evaluator ?? '', + evaluationSpecifics: '', + summary, + title: options.title ?? `Accessibility evaluation of ${site}`, + }, + }; +} + +/** Is this a report this command wrote? Enough of a check to say so in an error. */ +export function isReport(value: unknown): value is Report { + const candidate = value as Partial | null; + return ( + typeof candidate === 'object' && + candidate !== null && + candidate.tool === 'cli-tools wcag' && + Array.isArray(candidate.pages) && + typeof candidate.wcagVersion === 'string' && + typeof candidate.level === 'string' + ); +} + +export const OPEN_STEPS = `The WCAG-EM Report Tool is a web page with no command line, so the hand-off is: + + 1. wcag audit https://example.org --pages 8 writes wcag-report.json + 2. wcag report wcag-report.json writes evaluation.json + 3. open ${REPORT_TOOL_URL} + 4. "Open evaluation" in the menu, choose evaluation.json + +Steps 1 (scope) and 3 (sample) are then filled in, and step 4 (audit) holds +one assertion per page and criterion: "failed" where axe proved a failure, +"cannot tell" where it found something to look at or only checked a part. +Nothing is marked "passed" — that is the evaluator's call, made in the tool. +The tool keeps the evaluation in the browser and saves it back out as JSON +from step 5.`; diff --git a/test/wcag.test.ts b/test/wcag.test.ts new file mode 100644 index 0000000..7bb3929 --- /dev/null +++ b/test/wcag.test.ts @@ -0,0 +1,484 @@ +import { describe, expect, it } from 'vitest'; + +import { + CRITERIA, + EVALUATION_CONTEXT, + type PageResult, + type Report, + type RuleFinding, + axeTags, + browserEnv, + chooseSample, + chromeCandidates, + compareNums, + criteriaFor, + criteriaFromTags, + criterionFromTag, + criterionId, + discoverSample, + findChrome, + formatSummary, + hasFailures, + isReport, + normalizeUrl, + pageOutcomes, + pageTitles, + parseSitemap, + sameOriginLinks, + sitemapsFromRobots, + summarize, + toEvaluation, + toMarkdown, + totals, + withinTarget, +} from '../src/wcag.ts'; + +// --------------------------------------------------------------------------- +// Criteria and tags +// --------------------------------------------------------------------------- + +describe('the criteria table', () => { + it('carries WCAG 2.1 as 78 criteria and 2.2 as 86, like the report tool', () => { + expect(criteriaFor('2.1')).toHaveLength(78); + expect(criteriaFor('2.2')).toHaveLength(86); + }); + + it('drops 4.1.1 from 2.2 and adds the nine new ones', () => { + const nums22 = criteriaFor('2.2').map((criterion) => criterion.num); + const nums21 = criteriaFor('2.1').map((criterion) => criterion.num); + expect(nums21).toContain('4.1.1'); + expect(nums22).not.toContain('4.1.1'); + expect(nums22.filter((num) => !nums21.includes(num))).toEqual([ + '2.4.11', '2.4.12', '2.4.13', '2.5.7', '2.5.8', '3.2.6', '3.3.7', '3.3.8', '3.3.9', + ]); + }); + + it('uses the report tool ids, which differ for 2.5.5 between versions', () => { + const contrast = CRITERIA.find((criterion) => criterion.num === '1.4.3')!; + expect(criterionId(contrast, '2.2')).toBe('WCAG22:contrast-minimum'); + expect(criterionId(contrast, '2.1')).toBe('WCAG21:contrast-minimum'); + const target = CRITERIA.find((criterion) => criterion.num === '2.5.5')!; + expect(criterionId(target, '2.1')).toBe('WCAG21:target-size'); + expect(criterionId(target, '2.2')).toBe('WCAG22:target-size-enhanced'); + }); + + it('has no duplicate numbers', () => { + const nums = CRITERIA.map((criterion) => criterion.num); + expect(new Set(nums).size).toBe(nums.length); + }); + + it('sorts 1.4.3 before 1.4.10', () => { + expect(['1.4.10', '1.4.3', '2.1.1', '1.1.1'].sort(compareNums)).toEqual(['1.1.1', '1.4.3', '1.4.10', '2.1.1']); + }); + + it('nests the levels: AA includes A, AAA includes both', () => { + expect(withinTarget('A', 'AA')).toBe(true); + expect(withinTarget('AAA', 'AA')).toBe(false); + expect(withinTarget('AA', 'AAA')).toBe(true); + }); +}); + +describe('criterionFromTag', () => { + it('reads principle, guideline and criterion off an axe tag', () => { + expect(criterionFromTag('wcag111')).toBe('1.1.1'); + expect(criterionFromTag('wcag143')).toBe('1.4.3'); + expect(criterionFromTag('wcag1412')).toBe('1.4.12'); + expect(criterionFromTag('wcag2411')).toBe('2.4.11'); + }); + + it('ignores level and category tags', () => { + expect(criterionFromTag('wcag2aa')).toBeNull(); + expect(criterionFromTag('wcag21a')).toBeNull(); + expect(criterionFromTag('best-practice')).toBeNull(); + expect(criterionFromTag('cat.color')).toBeNull(); + }); + + // axe tags `wcag2a-obsolete` on the 4.1.1 rules and other tags name + // criteria WCAG does not have; neither should invent a row. + it('refuses a number that is not a criterion', () => { + expect(criterionFromTag('wcag199')).toBeNull(); + expect(criterionFromTag('wcag2a-obsolete')).toBeNull(); + }); + + it('collects the criteria of a rule in order, once each', () => { + expect(criteriaFromTags(['cat.color', 'wcag2aa', 'wcag143', 'wcag1411', 'wcag143', 'ACT'])).toEqual(['1.4.3', '1.4.11']); + }); +}); + +describe('axeTags', () => { + it('selects every version up to the target at every level within it', () => { + expect(axeTags('2.2', 'AA')).toEqual(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'wcag22a', 'wcag22aa']); + expect(axeTags('2.1', 'A')).toEqual(['wcag2a', 'wcag21a']); + expect(axeTags('2.2', 'AAA')).toContain('wcag2aaa'); + }); + + it('never asks for best practices', () => { + expect(axeTags('2.2', 'AAA')).not.toContain('best-practice'); + }); +}); + +// --------------------------------------------------------------------------- +// The sample +// --------------------------------------------------------------------------- + +describe('parseSitemap', () => { + it('reads a urlset', () => { + const xml = ` + https://example.org/2026-01-01 + https://example.org/about?a=1&b=2 + `; + expect(parseSitemap(xml)).toEqual({ urls: ['https://example.org/', 'https://example.org/about?a=1&b=2'], sitemaps: [] }); + }); + + it('reads an index into its children', () => { + const xml = `https://example.org/sitemap-1.xml + `; + expect(parseSitemap(xml)).toEqual({ urls: [], sitemaps: ['https://example.org/sitemap-1.xml', 'https://example.org/sitemap-2.xml'] }); + }); + + it('yields nothing for a page that is not a sitemap', () => { + expect(parseSitemap('404')).toEqual({ urls: [], sitemaps: [] }); + }); + + it('reads the sitemaps robots.txt names', () => { + expect(sitemapsFromRobots('User-agent: *\nDisallow: /admin\nSitemap: https://example.org/a.xml\nsitemap:https://example.org/b.xml\n')).toEqual([ + 'https://example.org/a.xml', + 'https://example.org/b.xml', + ]); + }); +}); + +describe('normalizeUrl', () => { + it('drops the fragment, index.html and a trailing slash', () => { + expect(normalizeUrl('https://example.org/docs/#top')).toBe('https://example.org/docs'); + expect(normalizeUrl('https://example.org/docs/index.html')).toBe('https://example.org/docs'); + expect(normalizeUrl('https://example.org/')).toBe('https://example.org/'); + }); + + it('refuses anything that is not a web page', () => { + expect(normalizeUrl('mailto:hi@example.org')).toBeNull(); + expect(normalizeUrl('not a url')).toBeNull(); + }); +}); + +describe('sameOriginLinks', () => { + const html = ``; + + it('keeps same-origin pages, absolute, in order, once each', () => { + expect(sameOriginLinks(html, 'https://example.org/start')).toEqual([ + 'https://example.org/', + 'https://example.org/about', + 'https://example.org/pricing', + 'https://example.org/docs?page=2&sort=asc', + ]); + }); +}); + +describe('chooseSample', () => { + const candidates = [ + 'https://example.org/posts/1', + 'https://example.org/posts/2', + 'https://example.org/posts/3', + 'https://example.org/about', + 'https://example.org/docs/intro', + 'https://example.org/docs/api', + 'https://example.org/pricing', + ]; + + it('starts with the start page and spreads across sections before repeating one', () => { + expect(chooseSample('https://example.org/', candidates, 5)).toEqual([ + 'https://example.org/', + 'https://example.org/posts/1', + 'https://example.org/about', + 'https://example.org/docs/intro', + 'https://example.org/pricing', + ]); + }); + + it('puts hand-picked pages before the crawl and never repeats one', () => { + expect(chooseSample('https://example.org/', candidates, 3, ['https://example.org/pricing', 'https://example.org/#x'])).toEqual([ + 'https://example.org/', + 'https://example.org/pricing', + 'https://example.org/posts/1', + ]); + }); + + it('is capped by the limit even when the extras alone exceed it', () => { + expect(chooseSample('https://example.org/', candidates, 2, ['https://example.org/a', 'https://example.org/b'])).toHaveLength(2); + }); +}); + +describe('discoverSample', () => { + const site: Record = { + 'https://example.org/robots.txt': 'Sitemap: https://example.org/extra.xml\n', + 'https://example.org/sitemap.xml': 'https://example.org/https://example.org/about', + 'https://example.org/extra.xml': 'https://example.org/pricinghttps://example.org/file.pdf', + 'https://example.org/': 'BlogCareers', + }; + const get = async (url: string): Promise => site[url] ?? null; + + it('reads the sitemap and the ones robots.txt adds, skipping files', async () => { + const sample = await discoverSample('https://example.org/', { pages: 10, method: 'sitemap', fetch: get }); + expect(sample.from).toBe('sitemap'); + expect(sample.pages).toEqual(['https://example.org/', 'https://example.org/about', 'https://example.org/pricing']); + expect(sample.candidates).toBe(3); + }); + + it('falls back to the start page links when auto finds too few', async () => { + const sample = await discoverSample('https://example.org/', { pages: 5, method: 'auto', fetch: get }); + expect(sample.pages).toEqual([ + 'https://example.org/', + 'https://example.org/about', + 'https://example.org/pricing', + 'https://example.org/blog', + 'https://example.org/careers', + ]); + }); + + it('audits only the start page when there is no sitemap and no links', async () => { + const sample = await discoverSample('https://example.org/', { pages: 5, method: 'auto', fetch: async () => null }); + expect(sample).toEqual({ pages: ['https://example.org/'], from: 'start', candidates: 0 }); + }); + + it('takes the list as given', async () => { + const sample = await discoverSample('https://example.org/', { pages: 5, method: 'list', extra: ['https://example.org/x'], fetch: get }); + expect(sample.pages).toEqual(['https://example.org/', 'https://example.org/x']); + expect(sample.from).toBe('list'); + }); +}); + +// --------------------------------------------------------------------------- +// Finding Chrome +// --------------------------------------------------------------------------- + +describe('findChrome', () => { + it('takes CHROME_PATH first, then PATH, then the caches', () => { + const candidates = chromeCandidates({ CHROME_PATH: '/x/chrome', PATH: '/usr/bin' }, '/home/nobody'); + expect(candidates[0]).toBe('/x/chrome'); + expect(candidates).toContain('/usr/bin/google-chrome'); + expect(candidates).toContain('/opt/google/chrome/chrome'); + }); + + it('reports none when nothing is executable', () => { + expect(findChrome({ PATH: '/nowhere' }, '/home/nobody', () => false)).toBeNull(); + }); + + it('returns the first executable candidate', () => { + expect(findChrome({ PATH: '/a:/b' }, '/home/nobody', (path) => path === '/b/chromium')).toBe('/b/chromium'); + }); + + it('leaves the environment alone when there is no staged library directory', () => { + const env = browserEnv({ PATH: '/usr/bin' }, '/nowhere/chrome-deps'); + expect(env).toEqual({ PATH: '/usr/bin' }); + }); +}); + +// --------------------------------------------------------------------------- +// Reading a report +// --------------------------------------------------------------------------- + +const finding = (rule: string, criteria: string[], nodes: number, impact: string | null = 'serious'): RuleFinding => ({ + rule, + impact, + help: `Help for ${rule}`, + helpUrl: `https://dequeuniversity.com/rules/axe/4.13/${rule}`, + criteria, + nodes, + targets: ['.hero > p', 'footer a'].slice(0, Math.min(2, nodes)), +}); + +const page = (url: string, title: string, overrides: Partial = {}): PageResult => ({ + url, + finalUrl: url, + title, + ok: true, + violations: [], + incomplete: [], + passes: [], + inapplicable: [], + ms: 100, + ...overrides, +}); + +const report = (): Report => ({ + tool: 'cli-tools wcag', + axeVersion: '4.13.0', + wcagVersion: '2.2', + level: 'AA', + site: 'https://example.org', + startedAt: '2026-09-13T10:00:00.000Z', + finishedAt: '2026-09-13T10:00:30.000Z', + sample: { from: 'sitemap', candidates: 40 }, + pages: [ + page('https://example.org/', 'Example', { + violations: [finding('color-contrast', ['1.4.3'], 18), finding('image-alt', ['1.1.1'], 2, 'critical')], + incomplete: [finding('color-contrast', ['1.4.3'], 3, null)], + passes: [finding('html-has-lang', ['3.1.1'], 1, null), finding('document-title', ['2.4.2'], 1, null)], + inapplicable: ['video-caption'], + }), + page('https://example.org/about', 'About', { + violations: [finding('color-contrast', ['1.4.3'], 4)], + incomplete: [finding('link-in-text-block', ['1.4.1'], 2, null)], + passes: [finding('html-has-lang', ['3.1.1'], 1, null)], + }), + page('https://example.org/pricing', 'Example', { + passes: [finding('html-has-lang', ['3.1.1'], 1, null), finding('color-contrast', ['1.4.3'], 40, null)], + }), + page('https://example.org/broken', '', { ok: false, error: 'net::ERR_NAME_NOT_RESOLVED' }), + ], +}); + +describe('pageOutcomes', () => { + it('fails on a violation, cannot tell otherwise, and never passes', () => { + const outcomes = pageOutcomes(report().pages[0]!); + expect(outcomes.get('1.4.3')?.outcome).toBe('failed'); + expect(outcomes.get('1.1.1')?.outcome).toBe('failed'); + expect(outcomes.get('3.1.1')?.outcome).toBe('cantTell'); + expect(outcomes.get('2.4.2')?.outcome).toBe('cantTell'); + expect(outcomes.has('1.2.2')).toBe(false); + }); +}); + +describe('summarize', () => { + it('counts pages per criterion and names the rule with the most elements', () => { + const rows = summarize(report()); + expect(rows.map((row) => row.num)).toEqual(['1.1.1', '1.4.1', '1.4.3', '2.4.2', '3.1.1']); + const contrast = rows.find((row) => row.num === '1.4.3')!; + expect(contrast).toMatchObject({ level: 'AA', failing: 2, review: 0, passing: 1, nodes: 22 }); + expect(contrast.worst).toEqual({ rule: 'color-contrast', nodes: 22, impact: 'serious' }); + expect(rows.find((row) => row.num === '1.4.1')).toMatchObject({ failing: 0, review: 1, passing: 0, worst: null }); + expect(rows.find((row) => row.num === '3.1.1')).toMatchObject({ failing: 0, review: 0, passing: 3 }); + }); + + it('skips pages that did not load', () => { + expect(totals(report())).toEqual({ pages: 4, loaded: 3, failingPages: 2, failingCriteria: 2, reviewCriteria: 1, nodes: 24 }); + }); + + it('only counts failures within the target', () => { + const strict = report(); + strict.level = 'A'; + expect(totals(strict).failingCriteria).toBe(1); + expect(hasFailures(strict)).toBe(true); + const clean = report(); + clean.pages = [page('https://example.org/', 'Example', { passes: [finding('html-has-lang', ['3.1.1'], 1, null)] })]; + expect(hasFailures(clean)).toBe(false); + }); +}); + +describe('formatSummary', () => { + it('prints a row per criterion and a totals line with the pages that broke', () => { + const text = formatSummary(report()); + expect(text).toContain('SC Level Criterion'); + expect(text).toMatch(/1\.4\.3\s+AA\s+Contrast \(Minimum\)\s+2\s+-\s+1\s+color-contrast \(22 elements, serious\)/); + expect(text).toContain('WCAG 2.2 level AA: 2 criteria failing on 2 pages of 3 (24 elements), 1 to review by hand. axe-core 4.13.0.'); + expect(text).toContain('could not load https://example.org/broken: net::ERR_NAME_NOT_RESOLVED'); + }); + + it('says one criterion, not one criteria', () => { + const one = report(); + one.pages = [page('https://example.org/', 'Example', { violations: [finding('image-alt', ['1.1.1'], 1)] })]; + expect(formatSummary(one)).toContain('1 criterion failing on 1 page of 1 (1 element)'); + }); +}); + +describe('toMarkdown', () => { + it('lists the sample, the failing rules under each criterion, and what needs a person', () => { + const md = toMarkdown(report()); + expect(md).toContain('# Accessibility audit of https://example.org'); + expect(md).toContain('- [Example](https://example.org/)'); + expect(md).toContain('- https://example.org/broken — could not load'); + expect(md).toContain('### 1.4.3 Contrast (Minimum) (AA) — fails on 2 pages'); + expect(md).toContain('**color-contrast** (serious): Help for color-contrast. [How to fix](https://dequeuniversity.com/rules/axe/4.13/color-contrast). 2 pages, e.g. `.hero > p`.'); + expect(md).toContain('- 1.4.1 Use of Color (A), 1 page'); + }); +}); + +// --------------------------------------------------------------------------- +// The report tool's evaluation file +// --------------------------------------------------------------------------- + +describe('pageTitles', () => { + it('tells two pages with the same title apart by path, and names an untitled page by its path', () => { + expect(pageTitles([page('https://example.org/', 'Blog'), page('https://example.org/blog', 'Blog'), page('https://example.org/x', ' ')])).toEqual([ + 'Blog', + 'Blog (/blog)', + '/x', + ]); + }); +}); + +describe('toEvaluation', () => { + const evaluation = toEvaluation(report(), { evaluator: 'Anthony' }) as Record; + + it('is the report tool\'s own shape: its context, type and five steps', () => { + expect(evaluation['@type']).toBe('Evaluation'); + expect(evaluation['@context']).toEqual(EVALUATION_CONTEXT); + // The tool's context has no WCAG22 term and its importer relies on that: + // a defined prefix expands the ids into IRIs it cannot fold back. + expect(evaluation['@context'].WCAG22).toBeUndefined(); + expect(Object.keys(evaluation)).toEqual(expect.arrayContaining(['defineScope', 'exploreTarget', 'selectSample', 'auditSample', 'reportFindings'])); + expect(evaluation.defineScope).toMatchObject({ wcagVersion: '2.2', conformanceTarget: 'AA' }); + expect(evaluation.defineScope.scope).toMatchObject({ type: ['TestSubject', 'Website'], title: 'example.org' }); + expect(evaluation.reportFindings.evaluator).toBe('Anthony'); + }); + + // The tool derives a sampled page's id from the URL in its description and + // finds it again by title, so both have to be exactly this. + it('puts every loaded page in the structured sample with its URL as id and description', () => { + const sample = evaluation.selectSample.structuredSample as Record[]; + expect(sample).toHaveLength(3); + expect(sample[0]).toEqual({ + id: 'https://example.org/', + type: ['TestSubject', 'Webpage'], + date: '2026-09-13T10:00:30.000Z', + title: 'Example', + description: 'https://example.org/', + }); + expect(sample[2]?.title).toBe('Example (/pricing)'); + expect(evaluation.selectSample.randomSample).toEqual([]); + }); + + it('writes one assertion per page and criterion, against the tool\'s criterion ids', () => { + const assertions = evaluation.auditSample as Record[]; + expect(assertions).toHaveLength(4 + 3 + 2); + const contrast = assertions.find((assertion) => assertion.subject.id === 'https://example.org/' && assertion.test.id === 'WCAG22:contrast-minimum'); + expect(contrast).toMatchObject({ + type: ['Assertion'], + mode: 'earl:automatic', + subject: { title: 'Example' }, + test: { type: ['TestCriterion', 'TestRequirement'], num: '1.4.3' }, + result: { outcome: { id: 'earl:failed', type: ['OutcomeValue', 'Fail'] } }, + }); + expect(contrast?.result.description).toContain('color-contrast [serious]: Help for color-contrast — 18 elements. e.g. .hero > p | footer a'); + const lang = assertions.find((assertion) => assertion.subject.id === 'https://example.org/' && assertion.test.id === 'WCAG22:language-of-page'); + expect(lang?.result.outcome.id).toBe('earl:cantTell'); + expect(lang?.result.description).toContain('Passing checks: html-has-lang.'); + expect(assertions.every((assertion) => assertion.result.outcome.id !== 'earl:passed')).toBe(true); + }); + + it('uses the 2.1 ids and leaves 2.2-only criteria out under 2.1', () => { + const older = report(); + older.wcagVersion = '2.1'; + older.pages = [page('https://example.org/', 'Example', { violations: [finding('target-size', ['2.5.8'], 1), finding('color-contrast', ['1.4.3'], 1)] })]; + const assertions = (toEvaluation(older) as Record).auditSample as Record[]; + expect(assertions.map((assertion) => assertion.test.id)).toEqual(['WCAG21:contrast-minimum']); + }); +}); + +describe('isReport', () => { + it('recognises what audit writes and nothing else', () => { + expect(isReport(report())).toBe(true); + expect(isReport({ pages: [] })).toBe(false); + expect(isReport(null)).toBe(false); + }); +});