Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## 2026-11-11

### Changes
- [Navigator] A step the Tester hands over with `interact` no longer counts as done just because its code ran without an error. If nothing on the page changed and the app sent no request of its own, the Navigator tries the next solution instead of reporting success, so a click on the wrong element is no longer reported back as the step being done. A change that shows up late, or a request to the app's own domain or subdomains, still counts; third-party calls such as analytics do not.
- Test plans: the Prerequisite URL of a plan keeps underscores (`?assets_host=…`, `/my_project/`). They were stripped before, so tests opened a different URL than the one written in the plan.
## 2026-11-12

Expand Down
18 changes: 13 additions & 5 deletions src/action.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
import type { Region } from './utils/region.js';
import { isInternalStep } from './utils/step-analyzer.ts';
import { safeFilename } from './utils/strings.ts';
import { isSameHostFamily } from './utils/url-matcher.js';
import { isAppRequest } from './utils/url-matcher.js';
import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';

const debugLog = createDebug('explorbot:action');
Expand All @@ -39,6 +39,7 @@ class Action {
public playwrightGroupId: string | null = null;
public assertionSteps: Array<{ name: string; args: any[] }> = [];
public executedSteps: ExecutedStep[] = [];
public sentRequests = 0;
public lastValue: unknown;
private recorder?: PlaywrightRecorder;
private recovery: RecoveryRunner;
Expand Down Expand Up @@ -293,6 +294,11 @@ class Action {

this.mainDocumentStatus = undefined;
this.networkRequests = [];
this.sentRequests = 0;

const onRequest = (request: any) => {
if (isAppRequest(request, this.baseOrigin)) this.sentRequests++;
};

const handler = (response: any) => {
const request = response.request();
Expand All @@ -309,17 +315,19 @@ class Action {
this.recordNetworkCall(request, status);
};

page.on('request', onRequest);
page.on('response', handler);
return () => page.off('response', handler);
return () => {
page.off('request', onRequest);
page.off('response', handler);
};
}

private recordNetworkCall(request: any, status: number): void {
const resourceType = request.resourceType();
if (resourceType !== 'xhr' && resourceType !== 'fetch') return;
if (!isAppRequest(request, this.baseOrigin)) return;

const url = URL.parse(request.url());
if (!url) return;
if (!isSameHostFamily(url.href, this.baseOrigin)) return;

const call: NetworkCall = { method: request.method(), path: url.pathname, status };
if (this.networkRequests.some((r) => r.method === call.method && r.path === call.path && r.status === call.status)) return;
Expand Down
33 changes: 30 additions & 3 deletions src/ai/navigator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { HooksRunner } from '../utils/hooks-runner.ts';
import { createDebug, pluralize, tag } from '../utils/logger.js';
import { loop, pause } from '../utils/loop.js';
import { RulesLoader } from '../utils/rules-loader.ts';
import { waitForAppRequest, waitForDomChange } from '../utils/page-readiness.ts';
import { normalizeInlineText } from '../utils/strings.ts';
import { extractStatePath, isSameHostFamily, matchesNavigationUrl } from '../utils/url-matcher.js';
import type { UrlFilter } from '../utils/url-filter.ts';
Expand All @@ -31,6 +32,7 @@ import { createLearnExperienceTool } from './tools.ts';

const debugLog = createDebug('explorbot:navigator');
const MAX_TRIED_STEPS = 5;
const SETTLE_MS = 1000;

class Navigator implements Agent {
emoji = '🧭';
Expand Down Expand Up @@ -339,9 +341,14 @@ class Navigator implements Agent {
progressBlocks.push(codeBlock);
}
if (freshHash !== prevHash) codeBlockIndex = codeBlocks.length;
} else {
resolved = attempt.ok;
if (attempt.ok) progressBlocks.push(codeBlock);
} else if (attempt.ok) {
resolved = await this.attemptChangedPage(action, prevActionResult);
if (resolved) progressBlocks.push(codeBlock);
if (!resolved) {
lastFailure = 'Ran without error but nothing on the page changed';
tag('warning').log(lastFailure);
batchFailures.push({ code: codeBlock, error: lastFailure });
}
}

if (!resolved) return;
Expand Down Expand Up @@ -513,6 +520,26 @@ class Navigator implements Agent {
return this.isSameExpectedOrigin(expectedUrl, action.stateManager) && matchesNavigationUrl(expectedUrl, this.comparableUrl(state, expectedUrl));
}

private async attemptChangedPage(action: Action, previous: ActionResult): Promise<boolean> {
if (action.sentRequests > 0) return true;
if (await this.pageChangedSince(previous)) return true;

const baseOrigin = URL.parse(this.config.playwright.url)?.origin || '';
const requestSent = await this.explorer.withPage((page) => {
const appRequest = waitForAppRequest(page, baseOrigin, SETTLE_MS);
const domChangeEndsWait = waitForDomChange(page, SETTLE_MS).then(() => false);
return Promise.race([appRequest, domChangeEndsWait]);
});
if (requestSent) return true;

return this.pageChangedSince(previous);
}

private async pageChangedSince(previous: ActionResult): Promise<boolean> {
const freshState = await this.explorer.capture();
return (await freshState.diff(previous)).hasChanges();
}

private async ariaDiff(freshState: ActionResult, previous: ActionResult): Promise<string | null> {
if (freshState.getStateHash() === previous.getStateHash()) return null;
try {
Expand Down
8 changes: 5 additions & 3 deletions src/ai/pilot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1093,12 +1093,14 @@ export class Pilot implements Agent {
if (resultMessage && (CHECK_TOOLS.includes(exec.toolName) || !exec.wasSuccessful)) {
line += `\n result: ${resultMessage}`;
}

groups[groups.length - 1].lines.push(`${index + 1}. ${line}`);

const messages = this.formatPageMessages(exec.output);
if (messages) line += `\n messages: ${messages}`;

groups.get(currentUrl)!.lines.push(line);
groups[groups.length - 1].lines.push(`${index + 1}. ${line}`);

const landedUrl = exec.output?.pageDiff?.currentUrl;
if (landedUrl && landedUrl !== currentUrl) startGroup(landedUrl);
}

const PER_GROUP_CAP = 25;
Expand Down
7 changes: 2 additions & 5 deletions src/api/xhr-capture.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { isSameHostFamily } from '../utils/url-matcher.js';
import { isAppRequest } from '../utils/url-matcher.js';
import { RequestResult, generateRequestId } from './request-result.ts';
import type { RequestStore } from './request-store.ts';

Expand Down Expand Up @@ -34,13 +34,10 @@ export class XhrCapture {

private async captureResponse(response: any): Promise<void> {
const request = response.request();
const resourceType = request.resourceType();

if (resourceType !== 'xhr' && resourceType !== 'fetch') return;
if (!isAppRequest(request, this.baseOrigin)) return;

const method = request.method();
const url = request.url();
if (!isSameHostFamily(url, this.baseOrigin)) return;

const status = response.status();

Expand Down
11 changes: 11 additions & 0 deletions src/utils/page-readiness.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { isAppRequest } from './url-matcher.ts';

export async function waitForPageReadiness(page: any, options: PageReadinessOptions = {}): Promise<boolean> {
if (!page) return false;

Expand Down Expand Up @@ -30,6 +32,15 @@ export function waitForDomChange(page: any, timeout: number): Promise<boolean> {
.catch(() => true);
}

export function waitForAppRequest(page: any, baseOrigin: string, timeout: number): Promise<boolean> {
if (!page?.waitForRequest) return Promise.resolve(false);

return page
.waitForRequest((request: any) => isAppRequest(request, baseOrigin), { timeout })
.then(() => true)
.catch(() => false);
}

const DOM_QUIET_MS = 350;

function waitForDomQuiet(page: any, timeout: number): Promise<void> {
Expand Down
6 changes: 6 additions & 0 deletions src/utils/url-matcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,3 +145,9 @@ export function isSameHostFamily(urlA: string, urlB: string): boolean {
if (!hostA || !hostB) return false;
return hostA === hostB || hostA.endsWith(`.${hostB}`) || hostB.endsWith(`.${hostA}`);
}

export function isAppRequest(request: any, baseOrigin: string): boolean {
const resourceType = request.resourceType();
if (resourceType !== 'xhr' && resourceType !== 'fetch') return false;
return isSameHostFamily(request.url(), baseOrigin);
}
2 changes: 1 addition & 1 deletion tests/unit/navigator-attempt-hook.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ describe('Navigator resolveState attempt hook', () => {
getActor: () => ({ wait: async () => {} }),
};

navigator.explorer = { action: () => action };
navigator.explorer = { action: () => action, capture: async () => ({ diff: async () => ({ hasChanges: () => true }) }) };
navigator.stateManager = action.stateManager;
return navigator;
}
Expand Down
66 changes: 63 additions & 3 deletions tests/unit/navigator-resolve-state.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,17 @@ function createHarness(
responses?: string[];
knowledge?: string;
ariaChanged?: string;
unchanged?: boolean;
sentRequests?: number;
settlePage?: any;
onSettle?: (page: { changed: boolean }) => void;
stopAt?: number;
stopReason?: string;
attempt?: (code: string, page: { url: string; hash: string }) => boolean;
onWait?: (page: { url: string; hash: string }) => void;
} = {}
) {
const page = { url: '/login', hash: 'start' };
const page = { url: '/login', hash: 'start', changed: !options.unchanged };
const sent: string[] = [];
const attempts: string[] = [];
const flows: string[] = [];
Expand All @@ -35,7 +39,7 @@ function createHarness(
url: page.url,
fullUrl: `${BASE_URL}${page.url}`,
getStateHash: () => page.hash,
diff: async () => ({ ariaChanged: options.ariaChanged ?? null }),
diff: async () => ({ ariaChanged: options.ariaChanged ?? null, hasChanges: () => page.changed }),
});

const action: any = {
Expand All @@ -45,6 +49,7 @@ function createHarness(
exitIframe: async () => {},
attempt: async (code: string) => {
attempts.push(code);
action.sentRequests = options.sentRequests ?? 0;
const ok = options.attempt ? options.attempt(code, page) : true;
action.lastError = ok ? null : new Error('element not visible\n at Object.<anonymous>');
return ok;
Expand All @@ -70,7 +75,14 @@ function createHarness(
return { response: { text: options.responses?.[index] ?? '' } };
},
};
navigator.explorer = { action: () => action, capture: async () => state() };
navigator.explorer = {
action: () => action,
capture: async () => state(),
withPage: async (fn: (settlePage: any) => Promise<any>) => {
options.onSettle?.(page);
return fn(options.settlePage ?? {});
},
};
navigator.stateManager = action.stateManager;

return { navigator, action, page, sent, attempts, flows };
Expand Down Expand Up @@ -144,6 +156,54 @@ describe('Navigator resolveState', () => {
expect(resolved).toBe(true);
});

it('does not resolve on a step that changed nothing on the page', async () => {
const harness = createHarness({
responses: ["```js\nI.click('Project')\n```", ''],
unchanged: true,
});

const resolved = await harness.navigator.resolveState('open project settings', fakeActionResult());

expect(resolved).toBe(false);
expect(harness.sent[1]).toContain('Ran without error but nothing on the page changed');
expect(harness.flows).toHaveLength(0);
});

it('resolves on a step that sent a request to the app even if the page did not change', async () => {
const harness = createHarness({ responses: ["```js\nI.click('Save')\n```"], unchanged: true, sentRequests: 1 });

expect(await harness.navigator.resolveState('save the form', fakeActionResult())).toBe(true);
});

it('resolves when the page changes only after the first capture', async () => {
const harness = createHarness({
responses: ["```js\nI.click('Save')\n```"],
unchanged: true,
onSettle: (page) => {
page.changed = true;
},
});

expect(await harness.navigator.resolveState('save the form', fakeActionResult())).toBe(true);
});

it('resolves when the app sends a request while the page settles, but not on a third-party request', async () => {
const settlePage = (url: string) => ({
evaluate: () => new Promise(() => {}),
waitForRequest: async (predicate: (request: any) => boolean) => {
const request = { resourceType: () => 'fetch', url: () => url };
if (predicate(request)) return request;
throw new Error('Timeout');
},
});

const app = createHarness({ responses: ["```js\nI.click('Save')\n```"], unchanged: true, settlePage: settlePage(`${BASE_URL}/api/save`) });
expect(await app.navigator.resolveState('save the form', fakeActionResult())).toBe(true);

const analytics = createHarness({ responses: ["```js\nI.click('Save')\n```", ''], unchanged: true, settlePage: settlePage('https://analytics.example.net/collect') });
expect(await analytics.navigator.resolveState('save the form', fakeActionResult())).toBe(false);
});

it('treats code blocks as alternatives when no URL is expected', async () => {
const harness = createHarness({
responses: ["```js\nI.click('a')\n```\n\n```js\nI.click('b')\n```"],
Expand Down
Loading