From cba04c6bbdc65cd09921b828749714b898f36fc8 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Wed, 11 Nov 2026 15:19:46 +0200 Subject: [PATCH 1/4] Navigator: a step that changed nothing does not resolve Without an expected URL, resolveState accepted the first code block that ran without an error. A click on the wrong element (same page, nothing changed) came back to the Tester as a successful interact(). Compare the page after the block with the page before it and treat an unchanged page as a failed attempt, fed back to the AI like the existing "page state did not change" case. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +++++ src/ai/navigator.ts | 12 +++++++++--- tests/unit/navigator-attempt-hook.test.ts | 2 +- tests/unit/navigator-resolve-state.test.ts | 16 +++++++++++++++- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecfdc6c5..92297944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 2026-11-11 + +### Changes +- [Navigator] A step the Tester hands over with `interact` no longer counts as done just because its code ran without an error. If nothing on the page changed, the Navigator tries the next solution instead of reporting success, so a click on the wrong element is no longer reported back as the step being done. + ## 2026-10-31 ### New CLI Options diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 834650a0..256e4ab3 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -338,9 +338,15 @@ class Navigator implements Agent { if (freshHash !== prevHash && (attempt.ok || check.urlMatches)) { progressBlocks.push(codeBlock); } - } else { - resolved = attempt.ok; - if (attempt.ok) progressBlocks.push(codeBlock); + } else if (attempt.ok) { + const freshState = await this.explorer.capture(); + resolved = (await freshState.diff(prevActionResult)).hasChanges(); + if (resolved) progressBlocks.push(codeBlock); + if (!resolved) { + lastFailure = 'Ran without error but nothing on the page changed'; + tag('warning').log(lastFailure); + batchFailures.push({ code: codeBlock, error: lastFailure }); + } } if (!resolved) return; diff --git a/tests/unit/navigator-attempt-hook.test.ts b/tests/unit/navigator-attempt-hook.test.ts index 3203b4af..898123d8 100644 --- a/tests/unit/navigator-attempt-hook.test.ts +++ b/tests/unit/navigator-attempt-hook.test.ts @@ -30,7 +30,7 @@ describe('Navigator resolveState attempt hook', () => { getActor: () => ({ wait: async () => {} }), }; - navigator.explorer = { action: () => action }; + navigator.explorer = { action: () => action, capture: async () => ({ diff: async () => ({ hasChanges: () => true }) }) }; navigator.stateManager = action.stateManager; return navigator; } diff --git a/tests/unit/navigator-resolve-state.test.ts b/tests/unit/navigator-resolve-state.test.ts index e8b28d68..1723dcfd 100644 --- a/tests/unit/navigator-resolve-state.test.ts +++ b/tests/unit/navigator-resolve-state.test.ts @@ -19,6 +19,7 @@ function createHarness( responses?: string[]; knowledge?: string; ariaChanged?: string; + unchanged?: boolean; stopAt?: number; stopReason?: string; attempt?: (code: string, page: { url: string; hash: string }) => boolean; @@ -35,7 +36,7 @@ function createHarness( url: page.url, fullUrl: `${BASE_URL}${page.url}`, getStateHash: () => page.hash, - diff: async () => ({ ariaChanged: options.ariaChanged ?? null }), + diff: async () => ({ ariaChanged: options.ariaChanged ?? null, hasChanges: () => !options.unchanged }), }); const action: any = { @@ -144,6 +145,19 @@ describe('Navigator resolveState', () => { expect(resolved).toBe(true); }); + it('does not resolve on a step that changed nothing on the page', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Project')\n```", ''], + unchanged: true, + }); + + const resolved = await harness.navigator.resolveState('open project settings', fakeActionResult()); + + expect(resolved).toBe(false); + expect(harness.sent[1]).toContain('Ran without error but nothing on the page changed'); + expect(harness.flows).toHaveLength(0); + }); + it('treats code blocks as alternatives when no URL is expected', async () => { const harness = createHarness({ responses: ["```js\nI.click('a')\n```\n\n```js\nI.click('b')\n```"], From 44323eb31758955fae7168c347aa6de783225e1c Mon Sep 17 00:00:00 2001 From: DavertMik Date: Thu, 12 Nov 2026 01:44:31 +0200 Subject: [PATCH 2/4] Navigator: a request to the app or a late change counts as a change Before calling a step "nothing changed", count it as done when it sent an XHR/fetch to the app's own domain (subdomains included, third parties such as analytics excluded), or when the page changes or the app sends a request within 1s after the first capture. Action now counts app requests as they start, so a slow request that has not answered yet is still seen. The "XHR/fetch to the app" check moves into isAppRequest() and is shared by Action, XhrCapture and the new waitForAppRequest(). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- src/action.ts | 18 +++++--- src/ai/navigator.ts | 13 +++++- src/api/xhr-capture.ts | 7 +-- src/utils/page-readiness.ts | 11 +++++ src/utils/url-matcher.ts | 6 +++ tests/unit/navigator-resolve-state.test.ts | 52 ++++++++++++++++++++-- 7 files changed, 93 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92297944..2b7e82c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 2026-11-11 ### Changes -- [Navigator] A step the Tester hands over with `interact` no longer counts as done just because its code ran without an error. If nothing on the page changed, the Navigator tries the next solution instead of reporting success, so a click on the wrong element is no longer reported back as the step being done. +- [Navigator] A step the Tester hands over with `interact` no longer counts as done just because its code ran without an error. If nothing on the page changed and the app sent no request of its own, the Navigator tries the next solution instead of reporting success, so a click on the wrong element is no longer reported back as the step being done. A change that shows up late, or a request to the app's own domain or subdomains, still counts; third-party calls such as analytics do not. ## 2026-10-31 diff --git a/src/action.ts b/src/action.ts index f6f4b775..a16574b4 100644 --- a/src/action.ts +++ b/src/action.ts @@ -18,7 +18,7 @@ import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; import type { Region } from './utils/region.js'; import { isInternalStep } from './utils/step-analyzer.ts'; import { safeFilename } from './utils/strings.ts'; -import { isSameHostFamily } from './utils/url-matcher.js'; +import { isAppRequest } from './utils/url-matcher.js'; import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts'; const debugLog = createDebug('explorbot:action'); @@ -39,6 +39,7 @@ class Action { public playwrightGroupId: string | null = null; public assertionSteps: Array<{ name: string; args: any[] }> = []; public executedSteps: ExecutedStep[] = []; + public sentRequests = 0; public lastValue: unknown; private recorder?: PlaywrightRecorder; private recovery: RecoveryRunner; @@ -293,6 +294,11 @@ class Action { this.mainDocumentStatus = undefined; this.networkRequests = []; + this.sentRequests = 0; + + const onRequest = (request: any) => { + if (isAppRequest(request, this.baseOrigin)) this.sentRequests++; + }; const handler = (response: any) => { const request = response.request(); @@ -309,17 +315,19 @@ class Action { this.recordNetworkCall(request, status); }; + page.on('request', onRequest); page.on('response', handler); - return () => page.off('response', handler); + return () => { + page.off('request', onRequest); + page.off('response', handler); + }; } private recordNetworkCall(request: any, status: number): void { - const resourceType = request.resourceType(); - if (resourceType !== 'xhr' && resourceType !== 'fetch') return; + if (!isAppRequest(request, this.baseOrigin)) return; const url = URL.parse(request.url()); if (!url) return; - if (!isSameHostFamily(url.href, this.baseOrigin)) return; const call: NetworkCall = { method: request.method(), path: url.pathname, status }; if (this.networkRequests.some((r) => r.method === call.method && r.path === call.path && r.status === call.status)) return; diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 256e4ab3..0b053b85 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -18,6 +18,7 @@ import { HooksRunner } from '../utils/hooks-runner.ts'; import { createDebug, pluralize, tag } from '../utils/logger.js'; import { loop, pause } from '../utils/loop.js'; import { RulesLoader } from '../utils/rules-loader.ts'; +import { waitForAppRequest, waitForDomChange } from '../utils/page-readiness.ts'; import { normalizeInlineText } from '../utils/strings.ts'; import { extractStatePath, isSameHostFamily, matchesNavigationUrl } from '../utils/url-matcher.js'; import type { UrlFilter } from '../utils/url-filter.ts'; @@ -31,6 +32,7 @@ import { createLearnExperienceTool } from './tools.ts'; const debugLog = createDebug('explorbot:navigator'); const MAX_TRIED_STEPS = 5; +const SETTLE_MS = 1000; class Navigator implements Agent { emoji = '🧭'; @@ -339,8 +341,10 @@ class Navigator implements Agent { progressBlocks.push(codeBlock); } } else if (attempt.ok) { - const freshState = await this.explorer.capture(); - resolved = (await freshState.diff(prevActionResult)).hasChanges(); + const baseOrigin = URL.parse(this.config.playwright.url)?.origin || ''; + resolved = action.sentRequests > 0 || (await this.pageChangedSince(prevActionResult)); + if (!resolved) resolved = await this.explorer.withPage((page) => Promise.race([waitForAppRequest(page, baseOrigin, SETTLE_MS), waitForDomChange(page, SETTLE_MS).then(() => false)])); + if (!resolved) resolved = await this.pageChangedSince(prevActionResult); if (resolved) progressBlocks.push(codeBlock); if (!resolved) { lastFailure = 'Ran without error but nothing on the page changed'; @@ -516,6 +520,11 @@ class Navigator implements Agent { return this.isSameExpectedOrigin(expectedUrl, action.stateManager) && matchesNavigationUrl(expectedUrl, this.comparableUrl(state, expectedUrl)); } + private async pageChangedSince(previous: ActionResult): Promise { + const freshState = await this.explorer.capture(); + return (await freshState.diff(previous)).hasChanges(); + } + private async ariaDiff(freshState: ActionResult, previous: ActionResult): Promise { if (freshState.getStateHash() === previous.getStateHash()) return null; try { diff --git a/src/api/xhr-capture.ts b/src/api/xhr-capture.ts index 5ad8677d..33147fa8 100644 --- a/src/api/xhr-capture.ts +++ b/src/api/xhr-capture.ts @@ -1,4 +1,4 @@ -import { isSameHostFamily } from '../utils/url-matcher.js'; +import { isAppRequest } from '../utils/url-matcher.js'; import { RequestResult, generateRequestId } from './request-result.ts'; import type { RequestStore } from './request-store.ts'; @@ -34,13 +34,10 @@ export class XhrCapture { private async captureResponse(response: any): Promise { const request = response.request(); - const resourceType = request.resourceType(); - - if (resourceType !== 'xhr' && resourceType !== 'fetch') return; + if (!isAppRequest(request, this.baseOrigin)) return; const method = request.method(); const url = request.url(); - if (!isSameHostFamily(url, this.baseOrigin)) return; const status = response.status(); diff --git a/src/utils/page-readiness.ts b/src/utils/page-readiness.ts index c542e285..afab5ca2 100644 --- a/src/utils/page-readiness.ts +++ b/src/utils/page-readiness.ts @@ -1,3 +1,5 @@ +import { isAppRequest } from './url-matcher.ts'; + export async function waitForPageReadiness(page: any, options: PageReadinessOptions = {}): Promise { if (!page) return false; @@ -30,6 +32,15 @@ export function waitForDomChange(page: any, timeout: number): Promise { .catch(() => true); } +export function waitForAppRequest(page: any, baseOrigin: string, timeout: number): Promise { + if (!page?.waitForRequest) return Promise.resolve(false); + + return page + .waitForRequest((request: any) => isAppRequest(request, baseOrigin), { timeout }) + .then(() => true) + .catch(() => false); +} + const DOM_QUIET_MS = 350; function waitForDomQuiet(page: any, timeout: number): Promise { diff --git a/src/utils/url-matcher.ts b/src/utils/url-matcher.ts index 0e6fe454..79f3fe97 100644 --- a/src/utils/url-matcher.ts +++ b/src/utils/url-matcher.ts @@ -145,3 +145,9 @@ export function isSameHostFamily(urlA: string, urlB: string): boolean { if (!hostA || !hostB) return false; return hostA === hostB || hostA.endsWith(`.${hostB}`) || hostB.endsWith(`.${hostA}`); } + +export function isAppRequest(request: any, baseOrigin: string): boolean { + const resourceType = request.resourceType(); + if (resourceType !== 'xhr' && resourceType !== 'fetch') return false; + return isSameHostFamily(request.url(), baseOrigin); +} diff --git a/tests/unit/navigator-resolve-state.test.ts b/tests/unit/navigator-resolve-state.test.ts index 1723dcfd..755a62b5 100644 --- a/tests/unit/navigator-resolve-state.test.ts +++ b/tests/unit/navigator-resolve-state.test.ts @@ -20,13 +20,16 @@ function createHarness( knowledge?: string; ariaChanged?: string; unchanged?: boolean; + sentRequests?: number; + settlePage?: any; + onSettle?: (page: { changed: boolean }) => void; stopAt?: number; stopReason?: string; attempt?: (code: string, page: { url: string; hash: string }) => boolean; onWait?: (page: { url: string; hash: string }) => void; } = {} ) { - const page = { url: '/login', hash: 'start' }; + const page = { url: '/login', hash: 'start', changed: !options.unchanged }; const sent: string[] = []; const attempts: string[] = []; const flows: string[] = []; @@ -36,7 +39,7 @@ function createHarness( url: page.url, fullUrl: `${BASE_URL}${page.url}`, getStateHash: () => page.hash, - diff: async () => ({ ariaChanged: options.ariaChanged ?? null, hasChanges: () => !options.unchanged }), + diff: async () => ({ ariaChanged: options.ariaChanged ?? null, hasChanges: () => page.changed }), }); const action: any = { @@ -46,6 +49,7 @@ function createHarness( exitIframe: async () => {}, attempt: async (code: string) => { attempts.push(code); + action.sentRequests = options.sentRequests ?? 0; const ok = options.attempt ? options.attempt(code, page) : true; action.lastError = ok ? null : new Error('element not visible\n at Object.'); return ok; @@ -71,7 +75,14 @@ function createHarness( return { response: { text: options.responses?.[index] ?? '' } }; }, }; - navigator.explorer = { action: () => action, capture: async () => state() }; + navigator.explorer = { + action: () => action, + capture: async () => state(), + withPage: async (fn: (settlePage: any) => Promise) => { + options.onSettle?.(page); + return fn(options.settlePage ?? {}); + }, + }; navigator.stateManager = action.stateManager; return { navigator, action, page, sent, attempts, flows }; @@ -158,6 +169,41 @@ describe('Navigator resolveState', () => { expect(harness.flows).toHaveLength(0); }); + it('resolves on a step that sent a request to the app even if the page did not change', async () => { + const harness = createHarness({ responses: ["```js\nI.click('Save')\n```"], unchanged: true, sentRequests: 1 }); + + expect(await harness.navigator.resolveState('save the form', fakeActionResult())).toBe(true); + }); + + it('resolves when the page changes only after the first capture', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Save')\n```"], + unchanged: true, + onSettle: (page) => { + page.changed = true; + }, + }); + + expect(await harness.navigator.resolveState('save the form', fakeActionResult())).toBe(true); + }); + + it('resolves when the app sends a request while the page settles, but not on a third-party request', async () => { + const settlePage = (url: string) => ({ + evaluate: () => new Promise(() => {}), + waitForRequest: async (predicate: (request: any) => boolean) => { + const request = { resourceType: () => 'fetch', url: () => url }; + if (predicate(request)) return request; + throw new Error('Timeout'); + }, + }); + + const app = createHarness({ responses: ["```js\nI.click('Save')\n```"], unchanged: true, settlePage: settlePage(`${BASE_URL}/api/save`) }); + expect(await app.navigator.resolveState('save the form', fakeActionResult())).toBe(true); + + const analytics = createHarness({ responses: ["```js\nI.click('Save')\n```", ''], unchanged: true, settlePage: settlePage('https://analytics.example.net/collect') }); + expect(await analytics.navigator.resolveState('save the form', fakeActionResult())).toBe(false); + }); + it('treats code blocks as alternatives when no URL is expected', async () => { const harness = createHarness({ responses: ["```js\nI.click('a')\n```\n\n```js\nI.click('b')\n```"], From 52bcc6b738dd2a33c802b9a34ab7728fea3afcb3 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Thu, 12 Nov 2026 02:07:03 +0200 Subject: [PATCH 3/4] Navigator: move the "did the attempt change the page" check into one method attemptChangedPage() runs the request, capture and settle checks as early returns in order, so the loop body only records progress or the failure. Same behaviour. Co-Authored-By: Claude Opus 5.5 --- src/ai/navigator.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 0b053b85..d9d2697e 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -341,10 +341,7 @@ class Navigator implements Agent { progressBlocks.push(codeBlock); } } else if (attempt.ok) { - const baseOrigin = URL.parse(this.config.playwright.url)?.origin || ''; - resolved = action.sentRequests > 0 || (await this.pageChangedSince(prevActionResult)); - if (!resolved) resolved = await this.explorer.withPage((page) => Promise.race([waitForAppRequest(page, baseOrigin, SETTLE_MS), waitForDomChange(page, SETTLE_MS).then(() => false)])); - if (!resolved) resolved = await this.pageChangedSince(prevActionResult); + resolved = await this.attemptChangedPage(action, prevActionResult); if (resolved) progressBlocks.push(codeBlock); if (!resolved) { lastFailure = 'Ran without error but nothing on the page changed'; @@ -520,6 +517,21 @@ class Navigator implements Agent { return this.isSameExpectedOrigin(expectedUrl, action.stateManager) && matchesNavigationUrl(expectedUrl, this.comparableUrl(state, expectedUrl)); } + private async attemptChangedPage(action: Action, previous: ActionResult): Promise { + if (action.sentRequests > 0) return true; + if (await this.pageChangedSince(previous)) return true; + + const baseOrigin = URL.parse(this.config.playwright.url)?.origin || ''; + const requestSent = await this.explorer.withPage((page) => { + const appRequest = waitForAppRequest(page, baseOrigin, SETTLE_MS); + const domChangeEndsWait = waitForDomChange(page, SETTLE_MS).then(() => false); + return Promise.race([appRequest, domChangeEndsWait]); + }); + if (requestSent) return true; + + return this.pageChangedSince(previous); + } + private async pageChangedSince(previous: ActionResult): Promise { const freshState = await this.explorer.capture(); return (await freshState.diff(previous)).hasChanges(); From 5798f06d2c6a6f4eead2eb34270afe7cc474024a Mon Sep 17 00:00:00 2001 From: DavertMik Date: Thu, 12 Nov 2026 02:27:34 +0200 Subject: [PATCH 4/4] Pilot: fix the session log broken by merging #300 and #303 The merge left both PRs' pushes in formatSessionLog: #303's numbered push into the current page group (with #300's messages appended after it), and #300's leftover groups.get(), which crashed now that groups is an array. Every Pilot session-log test failed on main. Append the messages before the one numbered push, and start the landed page's group after a navigating action, so the action stays under the page it ran on. Co-Authored-By: Claude Opus 5.5 --- src/ai/pilot.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index 9c5ef5fb..b0176b85 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -1094,11 +1094,13 @@ export class Pilot implements Agent { line += `\n result: ${resultMessage}`; } - groups[groups.length - 1].lines.push(`${index + 1}. ${line}`); const messages = this.formatPageMessages(exec.output); if (messages) line += `\n messages: ${messages}`; - groups.get(currentUrl)!.lines.push(line); + groups[groups.length - 1].lines.push(`${index + 1}. ${line}`); + + const landedUrl = exec.output?.pageDiff?.currentUrl; + if (landedUrl && landedUrl !== currentUrl) startGroup(landedUrl); } const PER_GROUP_CAP = 25;