ci(editorjs): run the VoiceOver suite on the merge queue

.github/actions/voiceover-tests/action.yml

@@ -0,0 +1,64 @@
+name: "VoiceOver tests"
+description: Run a package's Guidepup VoiceOver suite against a real screen reader
+inputs:
+  package-name:
+    description: 'Name of the package'
+    required: true
+  working-directory:
+    description: 'Package working directory'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: .nvmrc
+
+    - name: Setup environment
+      uses: ./.github/actions/setup
+
+    # `--ci` skips the prompts for manual steps a local machine would need.
+    - name: Configure the runner for screen reader automation
+      shell: bash
+      run: npx --yes @guidepup/setup setup --ci
+
+    # Separate from `setup` and also required: it installs the preferences disk image.
+    - name: Install screen reader assets
+      shell: bash
+      run: npx --yes @guidepup/setup install
+
+    - name: Resolve Playwright version
+      id: playwright-version
+      shell: bash
+      run: echo "version=$(yarn workspace ${{ inputs.package-name }} exec playwright --version | awk '{ print $2 }')" >> "$GITHUB_OUTPUT"
+
+    - name: Restore Playwright browsers
+      uses: actions/cache@v5
+      id: playwright-cache
+      with:
+        path: ~/Library/Caches/ms-playwright
+        key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}
+
+    # webkit only, matching playwright.voiceover.config.ts.
+    - name: Install Playwright webkit
+      if: ${{ steps.playwright-cache.outputs.cache-hit != 'true' }}
+      shell: bash
+      run: yarn workspace ${{ inputs.package-name }} exec playwright install webkit
+
+    # The suite's webServer builds the workspace dependencies first; the fixture loads their dist.
+    - name: Run VoiceOver tests
+      shell: bash
+      run: yarn workspace ${{ inputs.package-name }} run test:e2e:voiceover
+
+    # The transcript of what VoiceOver said is the whole diagnosis on a failure.
+    - name: Upload VoiceOver transcripts
+      if: ${{ !cancelled() }}
+      uses: actions/upload-artifact@v7
+      with:
+        name: voiceover-recordings
+        path: |
+          ${{ inputs.working-directory }}/test-results/
+          ${{ inputs.working-directory }}/recordings/
+        if-no-files-found: ignore
+        retention-days: 7

.github/workflows/editorjs.yml

@@ -22,3 +22,25 @@ jobs:
       include-e2e: true
     secrets:
       stryker_dashboard_api_key: ${{ secrets.STRYKER_DASHBOARD_API_KEY }}
+
+  # Merge queue only: ~8 minutes driving a real screen reader is too slow for every PR push.
+  # A skipped job reports success, so this never blocks a pull request.
+  voiceover:
+    name: VoiceOver
+    if: ${{ github.event_name == 'merge_group' }}
+    # A real screen reader needs a real macOS session; the standard runner is free on a public repo.
+    runs-on: macos-15
+    # Well above the ~8 minute runtime, so a wedged VoiceOver fails rather than hangs.
+    timeout-minutes: 60
+    # Only five macOS jobs run at once account-wide, so a superseded entry must not hold one.
+    concurrency:
+      group: voiceover-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Run VoiceOver tests
+        uses: ./.github/actions/voiceover-tests
+        with:
+          package-name: '@editorjs/editorjs'
+          working-directory: './packages/editorjs'

packages/editorjs/e2e/tests/voiceover.spec.ts

@@ -209,7 +209,11 @@ async function walkTo(
     }
   }
 
-  throw new Error(`VoiceOver did not reach an item matching ${pattern.toString()} within ${maxSteps} ${direction} steps`);
+  /** The route separates a cursor that swept past the target from one stalled against a wall. */
+  throw new Error(
+    `VoiceOver did not reach an item matching ${pattern.toString()} within ${maxSteps} ${direction} steps. `
+    + `It passed through: ${JSON.stringify(descriptions)}`
+  );
 }
 
 /**
@@ -1117,20 +1121,37 @@ test('Case 17: announces applied links as links', async ({ page, voiceOver }) =>
   expect(insideBlock.join(' | ')).toContain('link');
 });
 
+/** What a sweep found, kept alongside everything it passed through. */
+interface ReachableItems {
+  /** Distinct items matching the pattern, in the order the cursor met them. */
+  matches: string[];
+
+  /** Every stop the sweep made, matching or not. */
+  all: string[];
+}
+
 /**
  * Every distinct item matching `pattern` that VoiceOver's cursor reaches within `SCAN_STEPS`
  * forward steps.
  *
  * Returns the matches rather than a boolean so that asserting "nothing is reachable" fails with
  * the offending announcements in the message. A bare `toBe(false)` says only that something
  * matched, which leaves you guessing at whether the fault is the page or the pattern.
+ *
+ * `all` comes too: the baseline fails on an empty `matches`, which on its own says nothing.
  * @param voiceOver - Guidepup VoiceOver controller
  * @param pattern - matched against the current item's text at each stop
  */
-async function collectReachable(voiceOver: VoiceOverPlaywright, pattern: RegExp): Promise<string[]> {
+async function collectReachable(
+  voiceOver: VoiceOverPlaywright,
+  pattern: RegExp
+): Promise<ReachableItems> {
   const reachable = await sweep(voiceOver, SCAN_STEPS);
 
-  return [...new Set(reachable.filter(item => pattern.test(item)))];
+  return {
+    matches: [...new Set(reachable.filter(item => pattern.test(item)))],
+    all: reachable,
+  };
 }
 
 test('Case 18: does not let the cursor reach toolbox items filtered out by search', async ({ page, voiceOver }) => {
@@ -1145,30 +1166,31 @@ test('Case 18: does not let the cursor reach toolbox items filtered out by searc
 
   const menuItemPattern = /menu item/;
 
+  // Both sweeps anchor here: with the menu open, only the button reaches the items.
+  await findItem(voiceOver, /add block/i, 'previous');
+
   // Baseline. It also guards the real assertion below: if VoiceOver words menu items
   // differently than this expects, the test fails here instead of making "nothing reachable"
   // pass for the wrong reason.
-  await resetCursor(voiceOver);
-
   const beforeFiltering = await collectReachable(voiceOver, menuItemPattern);
 
-  expect(beforeFiltering.length).toBeGreaterThan(0);
+  expect(
+    beforeFiltering.matches.length,
+    `Nothing matched ${menuItemPattern}. VoiceOver announced: ${JSON.stringify(beforeFiltering.all)}`
+  ).toBeGreaterThan(0);
 
   // Opening the toolbox puts DOM focus in its search field; filling it filters the list.
   await page.getByRole('searchbox', { name: 'Search' }).fill('no such tool');
 
   await expect(menu.getByRole('menuitem')).toHaveCount(0);
 
-  // Back to the top rather than continuing from wherever the scan stopped - that item may be
-  // one of the ones just hidden, and VoiceOver would keep describing it from where it stands.
-  // The round trip inside resetCursor is what makes this reliable: `fill()` is a page-driven
-  // change, so without it VoiceOver can still be describing the unfiltered list.
-  await resetCursor(voiceOver);
+  // Back to the anchor; the sweep ends past the popover, and this is what shows VoiceOver the fill().
+  await findItem(voiceOver, /add block/i, 'previous');
 
   // ui-kit hides filtered items with a CSS class and sets the `hidden` attribute alongside it;
   // the attribute is what takes them out of the accessibility tree, so nothing matching should
   // remain reachable. If something does, the message below carries its announcement - the
   // answer to "is this the hidden item, or is the pattern matching something else entirely"
   // is not worth guessing at.
-  expect(await collectReachable(voiceOver, menuItemPattern)).toEqual([]);
+  expect((await collectReachable(voiceOver, menuItemPattern)).matches).toEqual([]);
 });

packages/editorjs/playwright.voiceover.config.ts

@@ -24,6 +24,8 @@ export default defineConfig({
   testMatch: /voiceover\.spec\.ts/,
   reporter: 'list',
   timeout: TEST_TIMEOUT_MS,
+  // Matches playwright.config.ts: screen reader timing shifts under a loaded runner.
+  retries: isCI ? 2 : 0,
   use: {
     ...screenReaderConfig.use,
     baseURL: `http://localhost:${PORT}`,