From dcf000cc19b7f71fff37a7c60444238d327a28bc Mon Sep 17 00:00:00 2001 From: Joan Leon Date: Sat, 14 Nov 2026 15:17:35 +0100 Subject: [PATCH] chore: test the CLI with Playwright 1.65, and pin it in the Copy prompt Playwright 1.65.1 brings Chromium 159. The prompt of a snippet page installs and runs the same version the CLI is tested with, so PLAYWRIGHT goes from 1.63 to 1.65. Chromium 159 does not register an unload handler unless the page sends Permissions-Policy: unload=(self), and a frame also needs allow="unload" from the page that embeds it. Two tests of Back-Forward-Cache failed because their pages no longer had the handler they test: the fixture sends the header and allows the frame. The snippet does not change: without a handler there is nothing that blocks the bfcache. --- CHANGELOG.md | 3 +++ cli/tests/e2e/bfcache.test.js | 8 +++++++- lib/snippet-prompt.mjs | 2 +- lib/snippet-prompt.test.mjs | 4 ++-- package-lock.json | 14 +++++++------- scripts/eval-prompts/grade.test.mjs | 18 +++++++++--------- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5c8fc31..855a342f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Changed +- The prompt that the Copy prompt button of a snippet page copies installs and runs Playwright 1.65, the version the CLI is tested with, which brings Chromium 159. That Chromium does not run `unload` handlers unless the page asks for them with `Permissions-Policy: unload=(self)`, so on a page that does not, Back-Forward-Cache no longer finds an `unload` handler to report. + --- ## [2.6.3] — 2026-11-08 diff --git a/cli/tests/e2e/bfcache.test.js b/cli/tests/e2e/bfcache.test.js index 0a85d3c6..eebfd7eb 100644 --- a/cli/tests/e2e/bfcache.test.js +++ b/cli/tests/e2e/bfcache.test.js @@ -8,6 +8,8 @@ const expression = loadSnippet("Loading/Back-Forward-Cache").trim().replace(/;\s let server; let base; +const UNLOAD_PAGES = ["/unload-prop", "/blocked", "/frame-blocked"]; + beforeAll( () => new Promise((resolve) => { @@ -15,13 +17,17 @@ beforeAll( const path = req.url.split("?")[0]; const headers = { "Content-Type": "text/html" }; if (path === "/no-store") headers["Cache-Control"] = "no-store"; + // Chromium 159 turns the unload event off unless the page asks for it: without this header + // a handler is not registered, window.onunload stays null and nothing blocks the bfcache. + // A frame also needs allow="unload" from the page that embeds it + if (UNLOAD_PAGES.includes(path)) headers["Permissions-Policy"] = "unload=(self)"; res.writeHead(200, headers); const pages = { "/clean": "

clean

", "/unload-prop": "

unload

", "/iframe": '

iframe

', "/no-store": "

no-store

", - "/blocked": '

blocked

', + "/blocked": '

blocked

', "/frame-clean": "

frame

", "/frame-blocked": '

frame

', }; diff --git a/lib/snippet-prompt.mjs b/lib/snippet-prompt.mjs index ae5e67e0..c3d60d2f 100644 --- a/lib/snippet-prompt.mjs +++ b/lib/snippet-prompt.mjs @@ -9,7 +9,7 @@ const REPO = 'https://github.com/nucliweb/webperf-snippets/blob/main' // The Playwright the CLI is tested with, the same in the install and in the run, so the browser the // first command installs is the one the second expects -export const PLAYWRIGHT = '1.63' +export const PLAYWRIGHT = '1.65' // Snippets an agent cannot run in a meaningful way through the CLI const EXCLUDED = new Set([ diff --git a/lib/snippet-prompt.test.mjs b/lib/snippet-prompt.test.mjs index eac922d5..369e73bd 100644 --- a/lib/snippet-prompt.test.mjs +++ b/lib/snippet-prompt.test.mjs @@ -60,8 +60,8 @@ test('the snippets an agent cannot run meaningfully are excluded and have no pro test('the prompt pins the CLI, names the snippet and links the docs, the source and the contract', () => { const p = prompt('Loading/TTFB-Sub-Parts', '/Loading/TTFB') - assert.match(p, /npx -y -p webperf-snippets@9\.9\.9 -p playwright@1\.63 webperf-snippets --snippet TTFB-Sub-Parts --json/) - assert.match(p, /npx -y playwright@1\.63 install chromium/) + assert.match(p, /npx -y -p webperf-snippets@9\.9\.9 -p playwright@1\.65 webperf-snippets --snippet TTFB-Sub-Parts --json/) + assert.match(p, /npx -y playwright@1\.65 install chromium/) assert.match(p, /warning .* is expected/) assert.match(p, /https:\/\/webperf-snippets\.net\/Loading\/TTFB\b/) assert.match(p, /https:\/\/github\.com\/nucliweb\/webperf-snippets\/blob\/main\/snippets\/Loading\/TTFB-Sub-Parts\.js/) diff --git a/package-lock.json b/package-lock.json index 28acbd34..2b9a80fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7188,13 +7188,13 @@ } }, "node_modules/playwright": { - "version": "1.63.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", - "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "version": "1.65.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.65.1.tgz", + "integrity": "sha512-vJ1FuoOrNwOKHUIVEvSt3jT1L+wjjDpwhwIstKtfvTh8XIG4FG4bIFuvNdMxi81LGfAhUgUZleVFaOLY4RIUUw==", "license": "Apache-2.0", "peer": true, "dependencies": { - "playwright-core": "1.63.0" + "playwright-core": "1.65.1" }, "bin": { "playwright": "cli.js" @@ -7204,9 +7204,9 @@ } }, "node_modules/playwright-core": { - "version": "1.63.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", - "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "version": "1.65.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.65.1.tgz", + "integrity": "sha512-hqPpz7blSMPxWVdoeE1yiBSzXsKiVOd2k+kjCvmUywD2GPO2Dd9HfcWnq9NN0GHbNZDFgZ9JfBFr+X5/BLo0qw==", "license": "Apache-2.0", "peer": true, "bin": { diff --git a/scripts/eval-prompts/grade.test.mjs b/scripts/eval-prompts/grade.test.mjs index df5e688a..7e8453b4 100644 --- a/scripts/eval-prompts/grade.test.mjs +++ b/scripts/eval-prompts/grade.test.mjs @@ -13,7 +13,7 @@ const result = (text) => JSON.stringify({ type: 'result', result: text, usage, t const PKG = 'webperf-snippets@0.4.1' const lcpJson = (value, rating = 'good') => JSON.stringify({ url: 'http://localhost/x', results: [{ id: 'LCP', script: 'LCP', status: 'ok', value, unit: 'ms', rating }] }, null, 2) -const lcpCommand = `npx -y -p ${PKG} -p playwright@1.63 webperf-snippets http://localhost/x --snippet LCP --json` +const lcpCommand = `npx -y -p ${PKG} -p playwright@1.65 webperf-snippets http://localhost/x --snippet LCP --json` const LCP_CASE = { snippet: 'LCP', expect: { status: ['ok'], reference: { value: 40, tolerance: 200 } } } @@ -41,7 +41,7 @@ test('cliRuns reads the JSON of a run even after npx warnings', () => { }) test('cliRuns ignores the install of Chromium and other commands', () => { - const t = parseTranscript([toolUse('a', 'Bash', { command: 'npx -y playwright@1.63 install chromium' }), toolResult('a', 'ok')].join('\n')) + const t = parseTranscript([toolUse('a', 'Bash', { command: 'npx -y playwright@1.65 install chromium' }), toolResult('a', 'ok')].join('\n')) assert.deepEqual(cliRuns(t, 'LCP'), []) }) @@ -120,7 +120,7 @@ test('the checks of the interactions file look at the file the agent left', () = expect: { status: ['ok', 'tracking'], noTyping: true, minInteractions: 1 }, } const inpJson = JSON.stringify({ results: [{ id: 'INP', status: 'ok', value: 224, rating: 'needs-improvement', details: { totalInteractions: 2 } }] }) - const command = `npx -y -p ${PKG} -p playwright@1.63 webperf-snippets http://localhost/x --snippet INP --interact-script interactions.json --json` + const command = `npx -y -p ${PKG} -p playwright@1.65 webperf-snippets http://localhost/x --snippet INP --interact-script interactions.json --json` const t = parseTranscript([toolUse('a', 'Bash', { command }), toolResult('a', inpJson), result('INP 224 ms, needs improvement')].join('\n')) const interactions = { interactions: [{ action: 'click', selector: '#load-more' }, { action: 'type', selector: 'input', text: 'x' }] } const checks = gradeRun(inpCase, t, { pkg: PKG, interactions, missingSelectors: ['input'] }) @@ -134,7 +134,7 @@ test('the checks of the interactions file look at the file the agent left', () = test('a score result is checked against its reference and in the report without a time unit', () => { const clsCase = { snippet: 'Layout-Shift-Loading-and-Interaction', interactive: true, expect: { status: ['ok'], reference: { value: 0.7606, tolerance: 0.05 } } } const clsJson = JSON.stringify({ results: [{ id: 'Layout-Shift-Loading-and-Interaction', status: 'ok', value: 0.7606, unit: 'score', rating: 'poor', details: { countedShifts: 3 } }] }) - const command = `npx -y -p ${PKG} -p playwright@1.63 webperf-snippets http://localhost/x --snippet Layout-Shift-Loading-and-Interaction --interact-script interactions.json --json` + const command = `npx -y -p ${PKG} -p playwright@1.65 webperf-snippets http://localhost/x --snippet Layout-Shift-Loading-and-Interaction --interact-script interactions.json --json` const t = parseTranscript([toolUse('a', 'Bash', { command }), toolResult('a', clsJson), result('CLS is 0.76, poor: three promo blocks push the text down.')].join('\n')) const interactions = { interactions: [{ action: 'scroll', y: 600 }, { action: 'wait', ms: 1000 }] } const ids = byId(gradeRun(clsCase, t, { pkg: PKG, interactions, missingSelectors: [] })) @@ -153,7 +153,7 @@ const BFCACHE_CASE = { }, } const bfcacheRun = (details, report) => { - const command = `npx -y -p ${PKG} -p playwright@1.63 webperf-snippets http://localhost/x --snippet Back-Forward-Cache --json` + const command = `npx -y -p ${PKG} -p playwright@1.65 webperf-snippets http://localhost/x --snippet Back-Forward-Cache --json` const json = JSON.stringify({ results: [{ id: 'Back-Forward-Cache', status: 'ok', details, issues: [] }] }) return parseTranscript([toolUse('a', 'Bash', { command }), toolResult('a', json), result(report)].join('\n')) } @@ -197,12 +197,12 @@ const NO_URL_CASE = { } test('a case that expects no run passes when the agent asks for the URL and runs nothing', () => { - const t = parseTranscript([toolUse('a', 'Bash', { command: 'npx -y playwright@1.63 install chromium' }), toolResult('a', 'ok'), result('Which URL do you want me to measure?')].join('\n')) + const t = parseTranscript([toolUse('a', 'Bash', { command: 'npx -y playwright@1.65 install chromium' }), toolResult('a', 'ok'), result('Which URL do you want me to measure?')].join('\n')) assert.deepEqual(byId(gradeRun(NO_URL_CASE, t, { pkg: PKG })), { 'no-cli-run': true, 'no-own-measurement': true, 'asks-for-url': true }) }) test('a case that expects no run fails when the agent measures a page it made up', () => { - const command = `npx -y -p ${PKG} -p playwright@1.63 webperf-snippets https://example.com --snippet LCP --json` + const command = `npx -y -p ${PKG} -p playwright@1.65 webperf-snippets https://example.com --snippet LCP --json` const t = parseTranscript([toolUse('a', 'Bash', { command }), toolResult('a', lcpJson(800)), result('LCP of example.com is 800 ms, good.')].join('\n')) const ids = byId(gradeRun(NO_URL_CASE, t, { pkg: PKG })) assert.equal(ids['no-cli-run'], false) @@ -219,7 +219,7 @@ const UNREACHABLE_CASE = { }, } const refused = (url) => `Error: page.goto: net::ERR_CONNECTION_REFUSED at ${url}` -const lcpOn = (url) => `npx -y -p ${PKG} -p playwright@1.63 webperf-snippets ${url} --snippet LCP --json` +const lcpOn = (url) => `npx -y -p ${PKG} -p playwright@1.65 webperf-snippets ${url} --snippet LCP --json` test('a case where the CLI fails passes when the agent reports the error of the URL it was given', () => { const t = parseTranscript([toolUse('a', 'Bash', { command: lcpOn(UNREACHABLE_URL) }), toolResult('a', refused(UNREACHABLE_URL)), result('The CLI failed: net::ERR_CONNECTION_REFUSED. Is the server running?')].join('\n')) @@ -328,7 +328,7 @@ test('a case with a page to fix checks the page the agent left in its directory' test('a prompt to fix is graded on a run before and one with --baseline, whose JSON goes to files, with the tarball by its name', () => { const fixCase = { snippet: 'LCP', expect: { status: ['ok'], fixed: { file: 'index.html', absent: /loading="lazy"/ } } } const tarball = '/tmp/eval/webperf-snippets-1.6.0.tgz' - const command = (extra) => `cd /tmp/eval; npx -y -p ./webperf-snippets-1.6.0.tgz -p playwright@1.63 webperf-snippets http://localhost/x --snippet LCP ${extra}` + const command = (extra) => `cd /tmp/eval; npx -y -p ./webperf-snippets-1.6.0.tgz -p playwright@1.65 webperf-snippets http://localhost/x --snippet LCP ${extra}` const t = parseTranscript([ toolUse('a', 'Bash', { command: command('--json > before.json') }), toolResult('a', ''), toolUse('b', 'Bash', { command: command('--baseline before.json --json > after.json') }), toolResult('b', ''),