diff --git a/.github/actions/voiceover-tests/action.yml b/.github/actions/voiceover-tests/action.yml new file mode 100644 index 00000000..e7b03e73 --- /dev/null +++ b/.github/actions/voiceover-tests/action.yml @@ -0,0 +1,64 @@ +name: "VoiceOver tests" +description: Run a package's Guidepup VoiceOver suite against a real screen reader +inputs: + package-name: + description: 'Name of the package' + required: true + working-directory: + description: 'Package working directory' + required: true +runs: + using: "composite" + steps: + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + + - name: Setup environment + uses: ./.github/actions/setup + + # `--ci` skips the prompts for manual steps a local machine would need. + - name: Configure the runner for screen reader automation + shell: bash + run: npx --yes @guidepup/setup setup --ci + + # Separate from `setup` and also required: it installs the preferences disk image. + - name: Install screen reader assets + shell: bash + run: npx --yes @guidepup/setup install + + - name: Resolve Playwright version + id: playwright-version + shell: bash + run: echo "version=$(yarn workspace ${{ inputs.package-name }} exec playwright --version | awk '{ print $2 }')" >> "$GITHUB_OUTPUT" + + - name: Restore Playwright browsers + uses: actions/cache@v5 + id: playwright-cache + with: + path: ~/Library/Caches/ms-playwright + key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }} + + # webkit only, matching playwright.voiceover.config.ts. + - name: Install Playwright webkit + if: ${{ steps.playwright-cache.outputs.cache-hit != 'true' }} + shell: bash + run: yarn workspace ${{ inputs.package-name }} exec playwright install webkit + + # The suite's webServer builds the workspace dependencies first; the fixture loads their dist. + - name: Run VoiceOver tests + shell: bash + run: yarn workspace ${{ inputs.package-name }} run test:e2e:voiceover + + # The transcript of what VoiceOver said is the whole diagnosis on a failure. + - name: Upload VoiceOver transcripts + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: voiceover-recordings + path: | + ${{ inputs.working-directory }}/test-results/ + ${{ inputs.working-directory }}/recordings/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/editorjs.yml b/.github/workflows/editorjs.yml index bf423d72..630014a9 100644 --- a/.github/workflows/editorjs.yml +++ b/.github/workflows/editorjs.yml @@ -22,3 +22,25 @@ jobs: include-e2e: true secrets: stryker_dashboard_api_key: ${{ secrets.STRYKER_DASHBOARD_API_KEY }} + + # Merge queue only: ~8 minutes driving a real screen reader is too slow for every PR push. + # A skipped job reports success, so this never blocks a pull request. + voiceover: + name: VoiceOver + if: ${{ github.event_name == 'merge_group' }} + # A real screen reader needs a real macOS session; the standard runner is free on a public repo. + runs-on: macos-15 + # Well above the ~8 minute runtime, so a wedged VoiceOver fails rather than hangs. + timeout-minutes: 60 + # Only five macOS jobs run at once account-wide, so a superseded entry must not hold one. + concurrency: + group: voiceover-${{ github.ref }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v6 + + - name: Run VoiceOver tests + uses: ./.github/actions/voiceover-tests + with: + package-name: '@editorjs/editorjs' + working-directory: './packages/editorjs' diff --git a/packages/editorjs/e2e/tests/voiceover.spec.ts b/packages/editorjs/e2e/tests/voiceover.spec.ts index 93e7ed21..807ef986 100644 --- a/packages/editorjs/e2e/tests/voiceover.spec.ts +++ b/packages/editorjs/e2e/tests/voiceover.spec.ts @@ -209,7 +209,11 @@ async function walkTo( } } - throw new Error(`VoiceOver did not reach an item matching ${pattern.toString()} within ${maxSteps} ${direction} steps`); + /** The route separates a cursor that swept past the target from one stalled against a wall. */ + throw new Error( + `VoiceOver did not reach an item matching ${pattern.toString()} within ${maxSteps} ${direction} steps. ` + + `It passed through: ${JSON.stringify(descriptions)}` + ); } /** @@ -1117,6 +1121,15 @@ test('Case 17: announces applied links as links', async ({ page, voiceOver }) => expect(insideBlock.join(' | ')).toContain('link'); }); +/** What a sweep found, kept alongside everything it passed through. */ +interface ReachableItems { + /** Distinct items matching the pattern, in the order the cursor met them. */ + matches: string[]; + + /** Every stop the sweep made, matching or not. */ + all: string[]; +} + /** * Every distinct item matching `pattern` that VoiceOver's cursor reaches within `SCAN_STEPS` * forward steps. @@ -1124,13 +1137,21 @@ test('Case 17: announces applied links as links', async ({ page, voiceOver }) => * Returns the matches rather than a boolean so that asserting "nothing is reachable" fails with * the offending announcements in the message. A bare `toBe(false)` says only that something * matched, which leaves you guessing at whether the fault is the page or the pattern. + * + * `all` comes too: the baseline fails on an empty `matches`, which on its own says nothing. * @param voiceOver - Guidepup VoiceOver controller * @param pattern - matched against the current item's text at each stop */ -async function collectReachable(voiceOver: VoiceOverPlaywright, pattern: RegExp): Promise { +async function collectReachable( + voiceOver: VoiceOverPlaywright, + pattern: RegExp +): Promise { const reachable = await sweep(voiceOver, SCAN_STEPS); - return [...new Set(reachable.filter(item => pattern.test(item)))]; + return { + matches: [...new Set(reachable.filter(item => pattern.test(item)))], + all: reachable, + }; } test('Case 18: does not let the cursor reach toolbox items filtered out by search', async ({ page, voiceOver }) => { @@ -1145,30 +1166,31 @@ test('Case 18: does not let the cursor reach toolbox items filtered out by searc const menuItemPattern = /menu item/; + // Both sweeps anchor here: with the menu open, only the button reaches the items. + await findItem(voiceOver, /add block/i, 'previous'); + // Baseline. It also guards the real assertion below: if VoiceOver words menu items // differently than this expects, the test fails here instead of making "nothing reachable" // pass for the wrong reason. - await resetCursor(voiceOver); - const beforeFiltering = await collectReachable(voiceOver, menuItemPattern); - expect(beforeFiltering.length).toBeGreaterThan(0); + expect( + beforeFiltering.matches.length, + `Nothing matched ${menuItemPattern}. VoiceOver announced: ${JSON.stringify(beforeFiltering.all)}` + ).toBeGreaterThan(0); // Opening the toolbox puts DOM focus in its search field; filling it filters the list. await page.getByRole('searchbox', { name: 'Search' }).fill('no such tool'); await expect(menu.getByRole('menuitem')).toHaveCount(0); - // Back to the top rather than continuing from wherever the scan stopped - that item may be - // one of the ones just hidden, and VoiceOver would keep describing it from where it stands. - // The round trip inside resetCursor is what makes this reliable: `fill()` is a page-driven - // change, so without it VoiceOver can still be describing the unfiltered list. - await resetCursor(voiceOver); + // Back to the anchor; the sweep ends past the popover, and this is what shows VoiceOver the fill(). + await findItem(voiceOver, /add block/i, 'previous'); // ui-kit hides filtered items with a CSS class and sets the `hidden` attribute alongside it; // the attribute is what takes them out of the accessibility tree, so nothing matching should // remain reachable. If something does, the message below carries its announcement - the // answer to "is this the hidden item, or is the pattern matching something else entirely" // is not worth guessing at. - expect(await collectReachable(voiceOver, menuItemPattern)).toEqual([]); + expect((await collectReachable(voiceOver, menuItemPattern)).matches).toEqual([]); }); diff --git a/packages/editorjs/playwright.voiceover.config.ts b/packages/editorjs/playwright.voiceover.config.ts index e3fbebf4..61066d8c 100644 --- a/packages/editorjs/playwright.voiceover.config.ts +++ b/packages/editorjs/playwright.voiceover.config.ts @@ -24,6 +24,8 @@ export default defineConfig({ testMatch: /voiceover\.spec\.ts/, reporter: 'list', timeout: TEST_TIMEOUT_MS, + // Matches playwright.config.ts: screen reader timing shifts under a loaded runner. + retries: isCI ? 2 : 0, use: { ...screenReaderConfig.use, baseURL: `http://localhost:${PORT}`,