diff --git a/CHANGELOG.md b/CHANGELOG.md index a2c11068..9c91a718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 2026-11-11 ### Changes +- [Navigator] A step the Tester hands over with `interact` no longer counts as done just because its code ran without an error. If nothing on the page changed and the app sent no request of its own, the Navigator tries the next solution instead of reporting success, so a click on the wrong element is no longer reported back as the step being done. A change that shows up late, or a request to the app's own domain or subdomains, still counts; third-party calls such as analytics do not. - Test plans: the Prerequisite URL of a plan keeps underscores (`?assets_host=…`, `/my_project/`). They were stripped before, so tests opened a different URL than the one written in the plan. ## 2026-11-12 diff --git a/src/action.ts b/src/action.ts index f6f4b775..a16574b4 100644 --- a/src/action.ts +++ b/src/action.ts @@ -18,7 +18,7 @@ import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; import type { Region } from './utils/region.js'; import { isInternalStep } from './utils/step-analyzer.ts'; import { safeFilename } from './utils/strings.ts'; -import { isSameHostFamily } from './utils/url-matcher.js'; +import { isAppRequest } from './utils/url-matcher.js'; import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts'; const debugLog = createDebug('explorbot:action'); @@ -39,6 +39,7 @@ class Action { public playwrightGroupId: string | null = null; public assertionSteps: Array<{ name: string; args: any[] }> = []; public executedSteps: ExecutedStep[] = []; + public sentRequests = 0; public lastValue: unknown; private recorder?: PlaywrightRecorder; private recovery: RecoveryRunner; @@ -293,6 +294,11 @@ class Action { this.mainDocumentStatus = undefined; this.networkRequests = []; + this.sentRequests = 0; + + const onRequest = (request: any) => { + if (isAppRequest(request, this.baseOrigin)) this.sentRequests++; + }; const handler = (response: any) => { const request = response.request(); @@ -309,17 +315,19 @@ class Action { this.recordNetworkCall(request, status); }; + page.on('request', onRequest); page.on('response', handler); - return () => page.off('response', handler); + return () => { + page.off('request', onRequest); + page.off('response', handler); + }; } private recordNetworkCall(request: any, status: number): void { - const resourceType = request.resourceType(); - if (resourceType !== 'xhr' && resourceType !== 'fetch') return; + if (!isAppRequest(request, this.baseOrigin)) return; const url = URL.parse(request.url()); if (!url) return; - if (!isSameHostFamily(url.href, this.baseOrigin)) return; const call: NetworkCall = { method: request.method(), path: url.pathname, status }; if (this.networkRequests.some((r) => r.method === call.method && r.path === call.path && r.status === call.status)) return; diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 2fcde8fa..fd41c1bd 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -18,6 +18,7 @@ import { HooksRunner } from '../utils/hooks-runner.ts'; import { createDebug, pluralize, tag } from '../utils/logger.js'; import { loop, pause } from '../utils/loop.js'; import { RulesLoader } from '../utils/rules-loader.ts'; +import { waitForAppRequest, waitForDomChange } from '../utils/page-readiness.ts'; import { normalizeInlineText } from '../utils/strings.ts'; import { extractStatePath, isSameHostFamily, matchesNavigationUrl } from '../utils/url-matcher.js'; import type { UrlFilter } from '../utils/url-filter.ts'; @@ -31,6 +32,7 @@ import { createLearnExperienceTool } from './tools.ts'; const debugLog = createDebug('explorbot:navigator'); const MAX_TRIED_STEPS = 5; +const SETTLE_MS = 1000; class Navigator implements Agent { emoji = '🧭'; @@ -339,9 +341,14 @@ class Navigator implements Agent { progressBlocks.push(codeBlock); } if (freshHash !== prevHash) codeBlockIndex = codeBlocks.length; - } else { - resolved = attempt.ok; - if (attempt.ok) progressBlocks.push(codeBlock); + } else if (attempt.ok) { + resolved = await this.attemptChangedPage(action, prevActionResult); + if (resolved) progressBlocks.push(codeBlock); + if (!resolved) { + lastFailure = 'Ran without error but nothing on the page changed'; + tag('warning').log(lastFailure); + batchFailures.push({ code: codeBlock, error: lastFailure }); + } } if (!resolved) return; @@ -513,6 +520,26 @@ class Navigator implements Agent { return this.isSameExpectedOrigin(expectedUrl, action.stateManager) && matchesNavigationUrl(expectedUrl, this.comparableUrl(state, expectedUrl)); } + private async attemptChangedPage(action: Action, previous: ActionResult): Promise { + if (action.sentRequests > 0) return true; + if (await this.pageChangedSince(previous)) return true; + + const baseOrigin = URL.parse(this.config.playwright.url)?.origin || ''; + const requestSent = await this.explorer.withPage((page) => { + const appRequest = waitForAppRequest(page, baseOrigin, SETTLE_MS); + const domChangeEndsWait = waitForDomChange(page, SETTLE_MS).then(() => false); + return Promise.race([appRequest, domChangeEndsWait]); + }); + if (requestSent) return true; + + return this.pageChangedSince(previous); + } + + private async pageChangedSince(previous: ActionResult): Promise { + const freshState = await this.explorer.capture(); + return (await freshState.diff(previous)).hasChanges(); + } + private async ariaDiff(freshState: ActionResult, previous: ActionResult): Promise { if (freshState.getStateHash() === previous.getStateHash()) return null; try { diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index 684bd222..b0176b85 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -1093,12 +1093,14 @@ export class Pilot implements Agent { if (resultMessage && (CHECK_TOOLS.includes(exec.toolName) || !exec.wasSuccessful)) { line += `\n result: ${resultMessage}`; } - - groups[groups.length - 1].lines.push(`${index + 1}. ${line}`); + const messages = this.formatPageMessages(exec.output); if (messages) line += `\n messages: ${messages}`; - groups.get(currentUrl)!.lines.push(line); + groups[groups.length - 1].lines.push(`${index + 1}. ${line}`); + + const landedUrl = exec.output?.pageDiff?.currentUrl; + if (landedUrl && landedUrl !== currentUrl) startGroup(landedUrl); } const PER_GROUP_CAP = 25; diff --git a/src/api/xhr-capture.ts b/src/api/xhr-capture.ts index 5ad8677d..33147fa8 100644 --- a/src/api/xhr-capture.ts +++ b/src/api/xhr-capture.ts @@ -1,4 +1,4 @@ -import { isSameHostFamily } from '../utils/url-matcher.js'; +import { isAppRequest } from '../utils/url-matcher.js'; import { RequestResult, generateRequestId } from './request-result.ts'; import type { RequestStore } from './request-store.ts'; @@ -34,13 +34,10 @@ export class XhrCapture { private async captureResponse(response: any): Promise { const request = response.request(); - const resourceType = request.resourceType(); - - if (resourceType !== 'xhr' && resourceType !== 'fetch') return; + if (!isAppRequest(request, this.baseOrigin)) return; const method = request.method(); const url = request.url(); - if (!isSameHostFamily(url, this.baseOrigin)) return; const status = response.status(); diff --git a/src/utils/page-readiness.ts b/src/utils/page-readiness.ts index c542e285..afab5ca2 100644 --- a/src/utils/page-readiness.ts +++ b/src/utils/page-readiness.ts @@ -1,3 +1,5 @@ +import { isAppRequest } from './url-matcher.ts'; + export async function waitForPageReadiness(page: any, options: PageReadinessOptions = {}): Promise { if (!page) return false; @@ -30,6 +32,15 @@ export function waitForDomChange(page: any, timeout: number): Promise { .catch(() => true); } +export function waitForAppRequest(page: any, baseOrigin: string, timeout: number): Promise { + if (!page?.waitForRequest) return Promise.resolve(false); + + return page + .waitForRequest((request: any) => isAppRequest(request, baseOrigin), { timeout }) + .then(() => true) + .catch(() => false); +} + const DOM_QUIET_MS = 350; function waitForDomQuiet(page: any, timeout: number): Promise { diff --git a/src/utils/url-matcher.ts b/src/utils/url-matcher.ts index 0e6fe454..79f3fe97 100644 --- a/src/utils/url-matcher.ts +++ b/src/utils/url-matcher.ts @@ -145,3 +145,9 @@ export function isSameHostFamily(urlA: string, urlB: string): boolean { if (!hostA || !hostB) return false; return hostA === hostB || hostA.endsWith(`.${hostB}`) || hostB.endsWith(`.${hostA}`); } + +export function isAppRequest(request: any, baseOrigin: string): boolean { + const resourceType = request.resourceType(); + if (resourceType !== 'xhr' && resourceType !== 'fetch') return false; + return isSameHostFamily(request.url(), baseOrigin); +} diff --git a/tests/unit/navigator-attempt-hook.test.ts b/tests/unit/navigator-attempt-hook.test.ts index 3203b4af..898123d8 100644 --- a/tests/unit/navigator-attempt-hook.test.ts +++ b/tests/unit/navigator-attempt-hook.test.ts @@ -30,7 +30,7 @@ describe('Navigator resolveState attempt hook', () => { getActor: () => ({ wait: async () => {} }), }; - navigator.explorer = { action: () => action }; + navigator.explorer = { action: () => action, capture: async () => ({ diff: async () => ({ hasChanges: () => true }) }) }; navigator.stateManager = action.stateManager; return navigator; } diff --git a/tests/unit/navigator-resolve-state.test.ts b/tests/unit/navigator-resolve-state.test.ts index 1d5a171e..42607821 100644 --- a/tests/unit/navigator-resolve-state.test.ts +++ b/tests/unit/navigator-resolve-state.test.ts @@ -19,13 +19,17 @@ function createHarness( responses?: string[]; knowledge?: string; ariaChanged?: string; + unchanged?: boolean; + sentRequests?: number; + settlePage?: any; + onSettle?: (page: { changed: boolean }) => void; stopAt?: number; stopReason?: string; attempt?: (code: string, page: { url: string; hash: string }) => boolean; onWait?: (page: { url: string; hash: string }) => void; } = {} ) { - const page = { url: '/login', hash: 'start' }; + const page = { url: '/login', hash: 'start', changed: !options.unchanged }; const sent: string[] = []; const attempts: string[] = []; const flows: string[] = []; @@ -35,7 +39,7 @@ function createHarness( url: page.url, fullUrl: `${BASE_URL}${page.url}`, getStateHash: () => page.hash, - diff: async () => ({ ariaChanged: options.ariaChanged ?? null }), + diff: async () => ({ ariaChanged: options.ariaChanged ?? null, hasChanges: () => page.changed }), }); const action: any = { @@ -45,6 +49,7 @@ function createHarness( exitIframe: async () => {}, attempt: async (code: string) => { attempts.push(code); + action.sentRequests = options.sentRequests ?? 0; const ok = options.attempt ? options.attempt(code, page) : true; action.lastError = ok ? null : new Error('element not visible\n at Object.'); return ok; @@ -70,7 +75,14 @@ function createHarness( return { response: { text: options.responses?.[index] ?? '' } }; }, }; - navigator.explorer = { action: () => action, capture: async () => state() }; + navigator.explorer = { + action: () => action, + capture: async () => state(), + withPage: async (fn: (settlePage: any) => Promise) => { + options.onSettle?.(page); + return fn(options.settlePage ?? {}); + }, + }; navigator.stateManager = action.stateManager; return { navigator, action, page, sent, attempts, flows }; @@ -144,6 +156,54 @@ describe('Navigator resolveState', () => { expect(resolved).toBe(true); }); + it('does not resolve on a step that changed nothing on the page', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Project')\n```", ''], + unchanged: true, + }); + + const resolved = await harness.navigator.resolveState('open project settings', fakeActionResult()); + + expect(resolved).toBe(false); + expect(harness.sent[1]).toContain('Ran without error but nothing on the page changed'); + expect(harness.flows).toHaveLength(0); + }); + + it('resolves on a step that sent a request to the app even if the page did not change', async () => { + const harness = createHarness({ responses: ["```js\nI.click('Save')\n```"], unchanged: true, sentRequests: 1 }); + + expect(await harness.navigator.resolveState('save the form', fakeActionResult())).toBe(true); + }); + + it('resolves when the page changes only after the first capture', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('Save')\n```"], + unchanged: true, + onSettle: (page) => { + page.changed = true; + }, + }); + + expect(await harness.navigator.resolveState('save the form', fakeActionResult())).toBe(true); + }); + + it('resolves when the app sends a request while the page settles, but not on a third-party request', async () => { + const settlePage = (url: string) => ({ + evaluate: () => new Promise(() => {}), + waitForRequest: async (predicate: (request: any) => boolean) => { + const request = { resourceType: () => 'fetch', url: () => url }; + if (predicate(request)) return request; + throw new Error('Timeout'); + }, + }); + + const app = createHarness({ responses: ["```js\nI.click('Save')\n```"], unchanged: true, settlePage: settlePage(`${BASE_URL}/api/save`) }); + expect(await app.navigator.resolveState('save the form', fakeActionResult())).toBe(true); + + const analytics = createHarness({ responses: ["```js\nI.click('Save')\n```", ''], unchanged: true, settlePage: settlePage('https://analytics.example.net/collect') }); + expect(await analytics.navigator.resolveState('save the form', fakeActionResult())).toBe(false); + }); + it('treats code blocks as alternatives when no URL is expected', async () => { const harness = createHarness({ responses: ["```js\nI.click('a')\n```\n\n```js\nI.click('b')\n```"],