From d1080d40957b7da6a6171912d3635f2c4109d14d Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 24 Oct 2026 23:47:28 -0400 Subject: [PATCH 1/9] fix: record native and widget selects, and give id-less save buttons a selector The interactive recorder stopped every recording at a native a textbox named after its name attribute. It now falls back to unique data-testid, data-test and name attributes, then a submit input's value, and names button inputs by their label, so id-less saves (Trac's input[name=submit], NetBox's button[name=_create]) carry the selector qualification needs. Found by the onboarding-effort benchmark (#1447). Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/BROWSER_RECORDING.md | 16 ++- .../backends/playwright_backend.py | 27 ++++ .../openadapt_flow/interactive_recorder.py | 59 +++++++- .../test_interactive_recorder_gestures.py | 136 ++++++++++++++++-- .../tests/test_structural_locator_idless.py | 121 ++++++++++++++++ 5 files changed, 339 insertions(+), 20 deletions(-) create mode 100644 packages/flow/tests/test_structural_locator_idless.py diff --git a/packages/flow/docs/BROWSER_RECORDING.md b/packages/flow/docs/BROWSER_RECORDING.md index f19b8a481a..5e83e7aed0 100644 --- a/packages/flow/docs/BROWSER_RECORDING.md +++ b/packages/flow/docs/BROWSER_RECORDING.md @@ -397,12 +397,20 @@ gesture compiles to one `DOUBLE_CLICK` step), right click, drag, text entry (including `contenteditable` and `role="textbox"`), special keys, keyboard shortcuts, and wheel scrolling. +A native `` selection. Its option commit happens in browser-native - dropdown UI that produces no recordable action events, so the choice would - be silently absent from the compiled workflow. The recording stops with an - explicit error at the moment of selection. +- A multiple-choice ``, + // ` + + + + + +""" + + +@pytest.fixture(scope="module") +def page_url(tmp_path_factory: pytest.TempPathFactory) -> str: + path = tmp_path_factory.mktemp("idless") / "newticket.html" + path.write_text(PAGE) + return path.as_uri() + + +@pytest.fixture(scope="module") +def browser() -> Iterator[Any]: + with sync_api.sync_playwright() as playwright: + launched = playwright.chromium.launch(headless=True) + try: + yield launched + finally: + launched.close() + + +def _center(page: Any, selector: str) -> tuple[int, int]: + box = page.locator(selector).first.bounding_box() + assert box is not None + return int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + + +def test_locator_names_idless_save_buttons(browser: Any, page_url: str) -> None: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + page.goto(page_url) + backend = PlaywrightBackend(page) + + trac = backend.structural_locator_at(*_center(page, "input[name=submit]")) + assert trac is not None + assert trac.selector == 'input[name="submit"]' + assert (trac.role, trac.name) == ("button", "Create ticket") + + netbox = backend.structural_locator_at(*_center(page, "button[name=_create]")) + assert netbox is not None + assert netbox.selector == 'button[name="_create"]' + assert (netbox.role, netbox.name) == ("button", "Create") + + # Two identical unnamed buttons: no selector that could pick the + # wrong one, only role and name. + twin = backend.structural_locator_at(*_center(page, "button:text('Save')")) + assert twin is not None + assert twin.selector is None + assert (twin.role, twin.name) == ("button", "Save") + finally: + page.close() + + +def test_recorded_create_step_carries_its_selector( + browser: Any, page_url: str, tmp_path: Path +) -> None: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + page.goto(page_url) + recorder = Recorder(PlaywrightBackend(page), tmp_path / "rec", app_url=page_url) + recorder.click(*_center(page, "input[name=field_summary]")) + recorder.type_text("Printer jams on tray 2", param="summary") + recorder.click(*_center(page, "input[name=submit]")) + recorder.finish() + assert page.evaluate("window.actions") == ["submit"] + finally: + page.close() + + workflow = compile_recording(tmp_path / "rec", tmp_path / "bundle", name="ticket") + save = workflow.steps[-1] + assert save.anchor is not None and save.anchor.structural is not None + assert save.anchor.structural.selector == 'input[name="submit"]' + assert save.anchor.structural.role == "button" + assert save.anchor.structural.name == "Create ticket" From 57946df981019d251028f8b4277ab1d780eb3baf Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 01:11:27 -0400 Subject: [PATCH 2/9] fix: a selector names the clicked button only when it matches it alone Review of #1537: the uniqueness check counted matches in `document` but never confirmed the single match was the clicked element. A button in a shadow root with a same-named look-alike in the page got the look-alike's selector, and a replay would click the wrong Save. The check now runs in the clicked element's own document and requires the one match to be that element, for the id too. Inside a shadow root no selector is built: Playwright's CSS locator pierces open shadow roots at replay, so a selector unique in one root could match a look-alike elsewhere. Co-Authored-By: Claude Opus 5.5 --- .../backends/playwright_backend.py | 21 ++++-- .../tests/test_structural_locator_idless.py | 73 +++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/packages/flow/openadapt_flow/backends/playwright_backend.py b/packages/flow/openadapt_flow/backends/playwright_backend.py index f7a30d7e0f..d6e1073f38 100644 --- a/packages/flow/openadapt_flow/backends/playwright_backend.py +++ b/packages/flow/openadapt_flow/backends/playwright_backend.py @@ -925,9 +925,22 @@ class _FramePoint: ) || el; const tag = actionable.tagName.toLowerCase(); let selector = null; + // A selector counts only when, in the clicked element's own document, + // it matches exactly one element and that element is the clicked one. A + // look-alike elsewhere must not make a selector "unique" for the wrong + // element. Inside a shadow root no selector is built: Playwright's CSS + // locator pierces open shadow roots at replay, so a selector unique in + // one root could match a look-alike in the page or another root. + const root = actionable.getRootNode(); + const unique = (candidate) => { + if (!(root instanceof Document)) return false; + try { + return root.querySelectorAll(candidate).length === 1 && + root.querySelector(candidate) === actionable; + } catch (_) { return false; } + }; const id = actionable.id; - if (id && document.querySelectorAll( - '#' + CSS.escape(id)).length === 1) { + if (id && unique('#' + CSS.escape(id))) { selector = '#' + CSS.escape(id); } // No unique id: the same unique attributes the interactive recorder uses, @@ -938,10 +951,6 @@ class _FramePoint: ? (actionable.getAttribute('type') || 'text').toLowerCase() : ''; const isButtonInput = ['submit', 'button', 'reset', 'image'] .indexOf(inputType) >= 0; - const unique = (candidate) => { - try { return document.querySelectorAll(candidate).length === 1; } - catch (_) { return false; } - }; for (const attr of ['data-testid', 'data-test', 'name']) { if (selector) break; const value = actionable.getAttribute(attr); diff --git a/packages/flow/tests/test_structural_locator_idless.py b/packages/flow/tests/test_structural_locator_idless.py index 12a69a52cb..d3b6d4c797 100644 --- a/packages/flow/tests/test_structural_locator_idless.py +++ b/packages/flow/tests/test_structural_locator_idless.py @@ -119,3 +119,76 @@ def test_recorded_create_step_carries_its_selector( assert save.anchor.structural.selector == 'input[name="submit"]' assert save.anchor.structural.role == "button" assert save.anchor.structural.name == "Create ticket" + + +SHADOW_PAGE = """ + +

+ + +""" + + +def test_a_lookalike_in_another_root_never_names_the_clicked_button( + browser: Any, tmp_path: Path +) -> None: + """A Save inside a shadow root, and a look-alike Save in the page with + the same name. ``button[name="save"]`` matches exactly one element in the + document, the look-alike, so a count alone would call it unique for the + shadow button and a replay would click the wrong Save.""" + + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + path = tmp_path / "shadow.html" + path.write_text(SHADOW_PAGE) + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + page.goto(path.as_uri()) + backend = PlaywrightBackend(page) + shadow = page.locator("save-panel").locator("button") + box = shadow.bounding_box() + assert box is not None + x, y = int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + + # The page's own hit test stops at the shadow host. + located = backend.structural_locator_at(x, y) + assert located is None or located.selector != 'button[name="save"]' + + # A hit test that reaches the button inside the shadow root (as a + # piercing one would): still no selector, because the only match in + # the document is the look-alike. + page.evaluate( + """() => { + const inner = document.querySelector('save-panel') + .shadowRoot.querySelector('button'); + document.elementFromPoint = () => inner; + }""" + ) + located = backend.structural_locator_at(x, y) + assert located is not None + assert located.selector is None + assert (located.role, located.name) == ("button", "Save") + + # The light-DOM Save is the one element its selector matches, and it + # is the clicked one: it keeps the selector. + page.reload() + light = page.locator("p > button") + box = light.bounding_box() + assert box is not None + located = backend.structural_locator_at( + int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + ) + assert located is not None and located.selector == 'button[name="save"]' + finally: + page.close() From 2f5a449a03899c483fb014d777ac532496ebce8d Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 3 Nov 2026 14:28:15 -0500 Subject: [PATCH 3/9] Address review: a select that lands on another option stops; no name selectors for buttons The read-back of a focused select is decisive: another label (type-ahead stopping on Mrs for Mr, or a value that is no option) never counts as landed. A choice whose label another option shares is refused at record time. A button's name attribute, and a bracketed row-position name, never become a selector; buttons keep their role and label. --- packages/flow/docs/BROWSER_RECORDING.md | 10 ++- .../backends/playwright_backend.py | 17 ++--- .../openadapt_flow/interactive_recorder.py | 20 +++-- .../flow/openadapt_flow/runtime/replayer.py | 19 +++-- .../test_interactive_recorder_gestures.py | 73 ++++++++++++++++++- .../tests/test_structural_locator_idless.py | 44 ++++++----- 6 files changed, 138 insertions(+), 45 deletions(-) diff --git a/packages/flow/docs/BROWSER_RECORDING.md b/packages/flow/docs/BROWSER_RECORDING.md index 5e83e7aed0..e56e142703 100644 --- a/packages/flow/docs/BROWSER_RECORDING.md +++ b/packages/flow/docs/BROWSER_RECORDING.md @@ -401,16 +401,18 @@ A native ``, or a select that holds a declared - secret. The recording stops at the moment of selection. +- A multiple-choice ``, - // `

+

" + ) + page.focus("#s") + outcome: Any + try: + outcome = PlaywrightBackend(page).select_focused_native_option(label) + except Exception as exc: # noqa: BLE001 - the refusal is the result + outcome = exc + value = page.evaluate("document.getElementById('s').value") + changes = page.evaluate("window.__changes") + finally: + browser.close() + return outcome, value, changes + + +_DECOY_TEXT = '' +_DECOY_ATTR = '' + + +@pytest.mark.parametrize("decoy", [_DECOY_TEXT, _DECOY_ATTR], ids=["text", "attr"]) +def test_native_select_choice_skips_a_whitespace_decoy(decoy: str) -> None: + # Playwright's select_option(label=) and option.label both collapse + # whitespace, so "Label Alpha" would pass for "Label Alpha". The match + # is on the raw label: the later exact option is the one chosen. + outcome, value, changes = _choose_in_select( + decoy + '', "Label Alpha" + ) + assert outcome is True + assert value == "v-alpha" + assert changes == 1 + + +@pytest.mark.parametrize( + "options", + [ + _DECOY_TEXT, + _DECOY_ATTR, + # Two exact duplicates: replay can't tell which was meant. + '' + '', + '', + '', + '', + '', + # The one exact option is disabled. + '', + ], + ids=[ + "decoy-text", + "decoy-attr", + "duplicates", + "leading", + "trailing", + "nbsp", + "attr-leading", + "disabled", + ], +) +def test_native_select_choice_without_one_exact_label_refuses(options: str) -> None: + from openadapt_flow.backend import StructuralResolutionRefused + + outcome, value, changes = _choose_in_select(options, "Label Alpha") + assert isinstance(outcome, StructuralResolutionRefused), outcome + assert value == "" + assert changes == 0 + + +@pytest.mark.parametrize( + ("variant", "label"), + [ + ('', " Label Alpha"), + ('', "Label Alpha "), + ('', "Label\u00a0Alpha"), + ], + ids=["leading", "trailing", "nbsp"], +) +def test_native_select_choice_matches_whitespace_variants_exactly( + variant: str, label: str +) -> None: + # Each variant is chosen only by its own exact string, never by the plain + # "Label Alpha" option beside it. + outcome, value, _changes = _choose_in_select( + '' + variant, label + ) + assert outcome is True + assert value == variant.split('"')[1] + + def test_native_select_with_a_shared_label_is_refused( fixture_url: str, tmp_path: Path ) -> None: From ee66bea7329a7be8468e333475d78346a9e4397c Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 8 Nov 2026 20:56:08 -0500 Subject: [PATCH 8/9] fix(flow): match native select labels with the recorder's normalization Replay compared raw option labels byte for byte while the recorder stores the label whitespace-squashed, so a formatted multi-line option recorded as "Dog" refused at replay. - NATIVE_OPTION_LABEL_JS (playwright_backend) is the one label function: option.label, runs of JavaScript whitespace made one space, trimmed, case kept. The interactive recorder's change listener now uses it in place of its inline copy, and replay matches each option through it. - Exactly one option must match and be enabled; zero, several (a "Label Alpha" beside "Label Alpha" is two) or a disabled match refuse with nothing delivered. The selectedIndex plus input/change delivery, the re-read of index, value and label, StructuralRefusalAfterDelivery and the guarded pass-through stay. - Tests: whitespace decoy and duplicates refuse, a disabled match refuses, a multi-line option selects, and one case per whitespace class shows the stored label (double space, tab, newline, leading/trailing, NBSP, em space, label attribute collapse; zero-width space and case stay). --- packages/flow/docs/BROWSER_RECORDING.md | 9 +- .../backends/playwright_backend.py | 58 +++---- .../openadapt_flow/interactive_recorder.py | 12 +- .../test_interactive_recorder_gestures.py | 151 +++++++++++------- 4 files changed, 135 insertions(+), 95 deletions(-) diff --git a/packages/flow/docs/BROWSER_RECORDING.md b/packages/flow/docs/BROWSER_RECORDING.md index 404df957d3..a0532ccc07 100644 --- a/packages/flow/docs/BROWSER_RECORDING.md +++ b/packages/flow/docs/BROWSER_RECORDING.md @@ -398,9 +398,12 @@ gesture compiles to one `DOUBLE_CLICK` step), right click, drag, text entry shortcuts, and wheel scrolling. A native ```, the one option whose raw label is -#: exactly ``label``, and fire ``input`` and ``change`` as -#: ``select_option`` does. Returns ``{kind: 'none'}`` when the target is no +#: A native option's label as the interactive recorder stores it and as +#: replay matches it: ``option.label`` (the label attribute, else the text), +#: every run of JavaScript whitespace (``\s``: tab, newline, no-break space +#: and the other Unicode spaces) made one space, then trimmed. Case is kept. +#: One function for both, so a recorded label and the option it names agree. +NATIVE_OPTION_LABEL_JS = ( + r"(o) => String(o.label || o.text || '').replace(/\s+/g, ' ').trim()" +) + +#: Choose, in a native single `` has no single enabled option labelled " "exactly as the step's value; refusing to choose one" ) - chosen = target.evaluate(_SELECTED_RAW_OPTION_JS) + chosen = target.evaluate(_SELECTED_OPTION_JS) if not ( isinstance(chosen, dict) and chosen.get("index") == outcome.get("index") diff --git a/packages/flow/openadapt_flow/interactive_recorder.py b/packages/flow/openadapt_flow/interactive_recorder.py index 2f5500ffba..5e524964ac 100644 --- a/packages/flow/openadapt_flow/interactive_recorder.py +++ b/packages/flow/openadapt_flow/interactive_recorder.py @@ -90,6 +90,7 @@ from openadapt_flow import sealed_set as _sealed from openadapt_flow.backends.playwright_backend import ( GUARDED_ACTIONABLE_SELECTOR, + NATIVE_OPTION_LABEL_JS, PlaywrightBackend, ) from openadapt_flow.ir import StructuralLocator, structural_locator_is_locatable @@ -2043,12 +2044,12 @@ def select_attached_page( if (!e.isTrusted) return; const fs = frameSpace(); if (fs === null) { refuseFrame(); return; } - // The label is what the select shows and what replay reads back - // (option.label: the label attribute, else the text). Replay can only - // choose by label, so a label two options share (case aside) is refused: - // type-ahead would pick the first of them. + // The label is what the select shows, whitespace squashed, and the one + // replay matches (NATIVE_OPTION_LABEL_JS, shared with the backend). + // Replay can only choose by label, so a label two options share (case + // aside) is refused: replay could not tell them apart. const option = el.multiple ? null : el.options[el.selectedIndex]; - const squashLabel = (o) => String(o.label || o.text || '').replace(/\s+/g, ' ').trim(); + const squashLabel = __NATIVE_OPTION_LABEL__; const text = option ? squashLabel(option) : ''; const sameLabel = text ? Array.from(el.options).filter( (o) => squashLabel(o).toLowerCase() === text.toLowerCase()).length : 0; @@ -2258,6 +2259,7 @@ def select_attached_page( .replace("__SUBMIT_FIELD__", json.dumps(SUBMIT_BINDING_FIELD_SELECTOR)) .replace("__OBSERVE_SUBMIT_BINDING__", OBSERVE_SUBMIT_BINDING_JS) .replace("__SECTION_CONTEXT__", SECTION_CONTEXT_JS) + .replace("__NATIVE_OPTION_LABEL__", NATIVE_OPTION_LABEL_JS) ) diff --git a/packages/flow/tests/test_interactive_recorder_gestures.py b/packages/flow/tests/test_interactive_recorder_gestures.py index 87c24c0744..e9eac9c975 100644 --- a/packages/flow/tests/test_interactive_recorder_gestures.py +++ b/packages/flow/tests/test_interactive_recorder_gestures.py @@ -319,12 +319,16 @@ def test_native_select_replay_refuses_a_missing_label( assert keys == 0 -def _choose_in_select(options_html: str, label: str) -> tuple[Any, str, int]: +def _choose_in_select(options_html: str, label: str) -> tuple[Any, str, int, list[str]]: """Focus ``#s`` built from ``options_html`` and ask the backend to choose ``label``: (the chooser's result or the exception it raised, the select's - value afterwards, how many change events fired).""" + value afterwards, how many change events fired, each option's label as + the recorder stores it).""" - from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.backends.playwright_backend import ( + NATIVE_OPTION_LABEL_JS, + PlaywrightBackend, + ) sync_api = pytest.importorskip("playwright.sync_api") with sync_api.sync_playwright() as pw: @@ -338,6 +342,10 @@ def _choose_in_select(options_html: str, label: str) -> tuple[Any, str, int]: "document.getElementById('s').addEventListener('change'," " () => { window.__changes += 1; });" ) + stored = page.evaluate( + "Array.from(document.getElementById('s').options).slice(1)" + f".map({NATIVE_OPTION_LABEL_JS})" + ) page.focus("#s") outcome: Any try: @@ -348,80 +356,105 @@ def _choose_in_select(options_html: str, label: str) -> tuple[Any, str, int]: changes = page.evaluate("window.__changes") finally: browser.close() - return outcome, value, changes + return outcome, value, changes, stored -_DECOY_TEXT = '' -_DECOY_ATTR = '' +def _assert_refused(outcome: Any, value: str, changes: int) -> None: + from openadapt_flow.backend import StructuralResolutionRefused + assert isinstance(outcome, StructuralResolutionRefused), outcome + assert value == "" + assert changes == 0 -@pytest.mark.parametrize("decoy", [_DECOY_TEXT, _DECOY_ATTR], ids=["text", "attr"]) -def test_native_select_choice_skips_a_whitespace_decoy(decoy: str) -> None: - # Playwright's select_option(label=) and option.label both collapse - # whitespace, so "Label Alpha" would pass for "Label Alpha". The match - # is on the raw label: the later exact option is the one chosen. - outcome, value, changes = _choose_in_select( - decoy + '', "Label Alpha" - ) - assert outcome is True - assert value == "v-alpha" - assert changes == 1 + +_EXACT = '' @pytest.mark.parametrize( - "options", + "decoy", [ - _DECOY_TEXT, - _DECOY_ATTR, - # Two exact duplicates: replay can't tell which was meant. - '' - '', - '', - '', - '', - '', - # The one exact option is disabled. - '', - ], - ids=[ - "decoy-text", - "decoy-attr", - "duplicates", - "leading", - "trailing", - "nbsp", - "attr-leading", - "disabled", + '', + '', ], + ids=["text", "attr"], ) -def test_native_select_choice_without_one_exact_label_refuses(options: str) -> None: - from openadapt_flow.backend import StructuralResolutionRefused +def test_native_select_choice_refuses_a_whitespace_decoy(decoy: str) -> None: + # Recording and replay squash whitespace alike, so "Label Alpha" and + # "Label Alpha" are one label: two matches, refused (the recorder refuses + # such a select too). Never a wrong-option success. + outcome, value, changes, stored = _choose_in_select(decoy + _EXACT, "Label Alpha") + assert stored == ["Label Alpha", "Label Alpha"] + _assert_refused(outcome, value, changes) + + +def test_native_select_choice_refuses_duplicates() -> None: + outcome, value, changes, _stored = _choose_in_select( + '' + '', + "Label Alpha", + ) + _assert_refused(outcome, value, changes) + + +def test_native_select_choice_refuses_a_disabled_match() -> None: + outcome, value, changes, _stored = _choose_in_select( + '', "Label Alpha" + ) + _assert_refused(outcome, value, changes) - outcome, value, changes = _choose_in_select(options, "Label Alpha") - assert isinstance(outcome, StructuralResolutionRefused), outcome - assert value == "" - assert changes == 0 +def test_native_select_choice_selects_a_multiline_option() -> None: + outcome, value, changes, stored = _choose_in_select( + '' + '', + "Dog", + ) + assert stored == ["Dog", "Cat"] + assert outcome is True + assert value == "v-dog" + assert changes == 1 + +# One case per whitespace class, as the recorder stores it (JavaScript \s, +# squashed and trimmed) and replay therefore matches it. @pytest.mark.parametrize( - ("variant", "label"), + ("option_html", "stored_label"), [ - ('', " Label Alpha"), - ('', "Label Alpha "), - ('', "Label\u00a0Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + # Zero-width space is not \s: it stays, so "Label Alpha" is no match. + ('', "Label\u200b Alpha"), + # Case is kept: the recorder stores it as shown. + ('', "label alpha"), + ], + ids=[ + "double-space", + "tab", + "newline", + "leading-trailing", + "nbsp", + "em-space", + "label-attr", + "zero-width-space", + "case", ], - ids=["leading", "trailing", "nbsp"], ) -def test_native_select_choice_matches_whitespace_variants_exactly( - variant: str, label: str +def test_native_select_choice_follows_the_recorders_whitespace( + option_html: str, stored_label: str ) -> None: - # Each variant is chosen only by its own exact string, never by the plain - # "Label Alpha" option beside it. - outcome, value, _changes = _choose_in_select( - '' + variant, label - ) - assert outcome is True - assert value == variant.split('"')[1] + outcome, value, changes, stored = _choose_in_select(option_html, "Label Alpha") + assert stored == [stored_label] + if stored_label == "Label Alpha": + assert outcome is True + assert value == "v" + assert changes == 1 + else: + _assert_refused(outcome, value, changes) def test_native_select_with_a_shared_label_is_refused( From 50b6abf359fbd7b1661e8af86c3e0270764e7060 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 8 Nov 2026 22:29:04 -0500 Subject: [PATCH 9/9] fix(flow): hold a native select choice to the option element chosen A page change handler could replace the chosen option with a same-label, same-index twin of another value; the check read the expected value after the events ran, so it agreed with the replacement and the step verified. - The page captures the chosen option element, its value and its label in a record before input and change fire, and the expected value is never read from the page after them. The choice holds only while that same element is connected, inside the select, its only selected option, with the captured value and label. That is checked right after the events (in the page) and again from Python; otherwise StructuralRefusalAfterDelivery, which now also marks the step's input unverified. - The backend keeps the record (native_select_choice_holds), and the replayer's typed-input read-back refuses when it no longer holds, so a handler that swaps the option later is caught too. Every text delivery clears the record. - Tests through record, compile and replay: replacing the option on change or on input, changing only its value, removing it, and replacing it after a timeout each halt the run with input unverified and no key events; a handler that edits unrelated DOM still verifies. --- .../backends/playwright_backend.py | 136 ++++++++++++------ .../flow/openadapt_flow/runtime/replayer.py | 9 ++ .../test_interactive_recorder_gestures.py | 104 ++++++++++++-- 3 files changed, 200 insertions(+), 49 deletions(-) diff --git a/packages/flow/openadapt_flow/backends/playwright_backend.py b/packages/flow/openadapt_flow/backends/playwright_backend.py index 2a0940a262..4d3fc27cc8 100644 --- a/packages/flow/openadapt_flow/backends/playwright_backend.py +++ b/packages/flow/openadapt_flow/backends/playwright_backend.py @@ -1169,10 +1169,12 @@ def _fill_iso_date(target: Any, text: str) -> bool: #: Choose, in a native single ````. Typing into a select only highlights an option where the browser opens a popup for it (Linux Chromium), and commits nothing until the popup closes; Enter to commit could submit a form. The option is chosen on the - element itself, by index, firing ``input`` and ``change`` with no key - events. The match is on the label as the recorder stores it + element itself, firing ``input`` and ``change`` with no key events. The + match is on the label as the recorder stores it (``NATIVE_OPTION_LABEL_JS``), so a multi-line ``