diff --git a/packages/flow/docs/BROWSER_RECORDING.md b/packages/flow/docs/BROWSER_RECORDING.md index f19b8a481..a0532ccc0 100644 --- a/packages/flow/docs/BROWSER_RECORDING.md +++ b/packages/flow/docs/BROWSER_RECORDING.md @@ -397,12 +397,26 @@ gesture compiles to one `DOUBLE_CLICK` step), right click, drag, text entry (including `contenteditable` and `role="textbox"`), special keys, keyboard shortcuts, and wheel scrolling. +A native `` selection. Its option commit happens in browser-native - dropdown UI that produces no recordable action events, so the choice would - be silently absent from the compiled workflow. The recording stops with an - explicit error at the moment of selection. +- A multiple-choice ```, the one option whose label +#: (``NATIVE_OPTION_LABEL_JS``) is exactly ``label``, and fire ``input`` and +#: ``change`` as ``select_option`` does. Returns a record: ``{kind: 'none'}`` +#: when the target is no native select; ``{kind: 'refused'}`` (nothing +#: changed) for a multiple or disabled select, zero or several matches, or a +#: disabled match; else ``{kind: 'selected'}`` with the chosen option ELEMENT, +#: its value and label captured BEFORE the events ran (a page handler cannot +#: reach this record), and ``held``: the choice check right after them. +_SELECT_EXACT_LABEL_JS = ( + r"""(e, label) => { + if (e.tagName !== 'SELECT') return {kind: 'none'}; + if (e.multiple || e.disabled) return {kind: 'refused'}; + const optionLabel = """ + + NATIVE_OPTION_LABEL_JS + + r"""; + const matches = Array.from(e.options).filter((o) => optionLabel(o) === label); + if (matches.length !== 1 || matches[0].disabled) return {kind: 'refused'}; + const option = matches[0]; + const record = { + kind: 'selected', select: e, option: option, + value: option.value, label: optionLabel(option), + }; + option.selected = true; + e.dispatchEvent(new Event('input', {bubbles: true, composed: true})); + e.dispatchEvent(new Event('change', {bubbles: true})); + record.held = (__HOLDS__)(record); + return record; +}""" +) + +#: Does a choice record still hold? The same option element (not one at the +#: same index or with the same label) is connected, inside the select, its +#: only selected option, with the value and label captured before delivery. +_NATIVE_CHOICE_HOLDS_JS = ( + r"""(r) => { + const optionLabel = """ + + NATIVE_OPTION_LABEL_JS + + r"""; + const e = r.select, o = r.option; + const selected = e.selectedOptions; + return e.isConnected && o.isConnected && e.contains(o) + && selected.length === 1 && selected[0] === o + && o.value === r.value && optionLabel(o) === r.label; +}""" +) +_SELECT_EXACT_LABEL_JS = _SELECT_EXACT_LABEL_JS.replace( + "__HOLDS__", _NATIVE_CHOICE_HOLDS_JS +) + + +def _select_native_option(target: Any, text: str) -> Optional[Any]: + """Choose the option labelled exactly ``text`` in a native `` has no single enabled option labelled " + "exactly as the step's value; refusing to choose one" + ) + if not ( + meta.get("label") == text + and meta.get("held") is True + and record.evaluate(_NATIVE_CHOICE_HOLDS_JS) is True + ): + raise StructuralRefusalAfterDelivery( + "the native and its kin are buttons named by their value, @@ -1897,12 +2018,28 @@ def _fill_iso_date(target: Any, text: str) -> bool: const inputType = (actionable.getAttribute('type') || '').toLowerCase(); const inputButton = tag === 'input' && ['submit', 'button', 'reset', 'image'].includes(inputType); + // No unique id: the same unique attributes the interactive recorder uses. + // A button's `name` alone names the form field it submits, not the + // button: `button[name=action]` would find Deny where Approve was + // recorded, so a button keeps its role and label. A bracketed name + // (`items[0][qty]`) is a row position, which a reordered list moves to + // another row. Neither is a selector. + const buttonLike = inputButton || tag === 'button'; + for (const attr of ['data-testid', 'data-test', 'name']) { + if (selector) break; + const value = actionable.getAttribute(attr); + if (!value) continue; + if (attr === 'name' && (buttonLike || /\[/.test(value))) continue; + const candidate = tag + '[' + attr + '="' + CSS.escape(value) + '"]'; + if (unique(candidate)) selector = candidate; + } let role = actionable.getAttribute('role'); if (!role) { const map = {button: 'button', a: 'link', input: 'textbox', select: 'combobox', textarea: 'textbox'}; role = inputButton ? 'button' : (map[tag] || null); if (tag === 'a' && !actionable.getAttribute('href')) role = null; + if (inputType === 'checkbox' || inputType === 'radio') role = inputType; } const fieldLabel = """ + _FIELD_LABEL_JS @@ -2213,6 +2350,10 @@ def __init__( # references only) for submit bindings. A new backend starts empty. self._fill_store_key = f"__oaflow_filled_{uuid.uuid4().hex}" self._fill_value_slot = f"__oaflow_filled_values_{uuid.uuid4().hex}" + # The last native ``, with no keystrokes; ``False`` (nothing delivered) + when it is not one. A label no single option carries raises + :class:`StructuralResolutionRefused` (see + :func:`_select_native_option`). The replayer calls this for a type + step before typing; recording and other callers keep ``type_text``. + """ + + self._native_select_choice = None + self._assert_qualification_environment_current() + try: + target = self._focused_element_handle() + is_select = target is not None and target.evaluate( + "e => e.tagName === 'SELECT'" + ) + except Exception: # noqa: BLE001 - unreadable focus: not a select + return False + if is_select is not True: + return False + # As type_text does: submit bindings and create-form identity checks + # see the select as a field this program filled. + self._register_focused_field() + self._choose_native_option(target, text) + self._snapshot_focused_field(text) + return True + + def _choose_native_option(self, target: Any, text: str) -> bool: + """:func:`_select_native_option`, keeping its choice record for + :meth:`native_select_choice_holds`.""" + + self._native_select_choice = None + record = _select_native_option(target, text) + if record is None: + return False + self._native_select_choice = record + return True + + def native_select_choice_holds(self) -> Optional[bool]: + """Does the last native `` commits its option through browser-native dropdown UI - // that produces no recordable action events, so the demonstrated choice - // would be silently absent from the compiled workflow and replay would - // proceed with whatever value the field happens to hold. Refuse loudly at - // the moment the operator makes the selection. + // that produces no other recordable event, so the choice is recorded + // here, as the option's label typed into the select. Replay types it and + // reads the select's chosen label back (focused_select_label_at). No + // Enter follows: Enter on a select inside a form submits the form. const el = deepEventTarget(e); - if (el && el.matches && el.matches('select')) { + if (!el || !el.matches || !el.matches('select')) return; + // A change the page's own script dispatched (a TomSelect or Select2 + // widget updating its hidden select) is an echo of clicks and typing + // already recorded on the visible widget. + if (!e.isTrusted) return; + const fs = frameSpace(); + if (fs === null) { refuseFrame(); return; } + // The label is what the select shows, whitespace squashed, and the one + // replay matches (NATIVE_OPTION_LABEL_JS, shared with the backend). + // Replay can only choose by label, so a label two options share (case + // aside) is refused: replay could not tell them apart. + const option = el.multiple ? null : el.options[el.selectedIndex]; + const squashLabel = __NATIVE_OPTION_LABEL__; + const text = option ? squashLabel(option) : ''; + const sameLabel = text ? Array.from(el.options).filter( + (o) => squashLabel(o).toLowerCase() === text.toLowerCase()).length : 0; + if (!text || sameLabel !== 1 || secretStateForInput(el)) { emit({kind: 'control_refusal', control: 'select'}); + return; } + const localRect = el.getBoundingClientRect(); + identityWithheldReason = null; + const rawField = elementName(el) || el.id || null; + const field = identityTextOrNull(rawField); + const o = { + kind: 'select', + field: field, + label: fieldLabel(el), + value: text, + __oaflow_input_session: inputSessionFor(el), + rect: [Math.round(localRect.left + fs.dx), Math.round(localRect.top + fs.dy), + Math.round(localRect.width), Math.round(localRect.height)], + }; + if (rawField !== null && field === null) { + o.identity_withheld = identityWithheldReason; + } + emitInFrameSpace(o); }); listen('focusout', (e) => commitSecretValueFor(inputEventTarget(e))); listen('submit', commitSecretValues); @@ -2195,6 +2233,12 @@ def select_attached_page( return; } if (SPECIAL.indexOf(e.key) < 0) return; + // Keys that move or commit a native select's choice: the change event + // records the chosen option. An Enter that submits the form shows up as + // the click on the form's button. + if (el && el.matches && el.matches('select') && SELECT_KEYS.indexOf(e.key) >= 0) { + return; + } emitInFrameSpace({kind: 'key', key: e.key}); }); @@ -2215,6 +2259,7 @@ def select_attached_page( .replace("__SUBMIT_FIELD__", json.dumps(SUBMIT_BINDING_FIELD_SELECTOR)) .replace("__OBSERVE_SUBMIT_BINDING__", OBSERVE_SUBMIT_BINDING_JS) .replace("__SECTION_CONTEXT__", SECTION_CONTEXT_JS) + .replace("__NATIVE_OPTION_LABEL__", NATIVE_OPTION_LABEL_JS) ) @@ -3422,10 +3467,11 @@ def _enqueue_browser_event( if kind == "control_refusal": control = str(event.get("control") or "control") self._listener_error = BrowserAttachError( - f"a native <{control}> selection is not a qualified browser " - "recording action; recording stopped so the demonstrated " - "selection cannot be silently absent from the compiled " - "workflow" + f"this native <{control}> choice can't be recorded (a " + "multiple select, a secret field, or a label two options " + "share, which replay can't tell apart); recording stopped so " + "the demonstrated choice cannot be silently absent from the " + "compiled workflow" ) self.done = True return @@ -3442,7 +3488,7 @@ def _enqueue_browser_event( self.done = True return raw_input_session = event.pop("__oaflow_input_session", None) - if kind == "input": + if kind in ("input", "select"): expected_prefix = f"{self._session_id}:input:" if ( not isinstance(raw_input_session, str) @@ -3511,6 +3557,7 @@ def _enqueue_browser_event( "right_click", "drag", "input", + "select", "key", "hotkey", "scroll", @@ -3913,7 +3960,7 @@ def _validate_event_batch(batch: list[dict[str, Any]]) -> None: and same_point ): return - if kinds == {"input"}: + if kinds == {"input"} or kinds == {"select"}: sessions = {event.get("_oaflow_input_session") for event in batch} fields = {event.get("field") for event in batch} if len(sessions) == 1 and None not in sessions and len(fields) == 1: @@ -4007,6 +4054,14 @@ def _process(self, ev: dict[str, Any]) -> None: self._flush_type() self._flush_scroll() self._record_key(ev) + elif kind == "select": + # A native select's choice: its label typed into the select (see + # the page's change listener). Type-ahead changes the choice once + # per letter; like keystrokes in one field, the run keeps the + # last one. + self._flush_click() + self._flush_scroll() + self._accumulate_input({**ev, "kind": "input", "secret": False}) def _flush_click(self) -> None: pending = self._pending_click diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index bec74341e..ff7cfcea8 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -89,6 +89,7 @@ SelectOptionBackend, StructuralBackend, StructuralElementNotActionable, + StructuralRefusalAfterDelivery, StructuralResolutionRefused, SubmitBindingBackend, ) @@ -12346,6 +12347,22 @@ def _settle_seeded(self) -> bytes: ) return result.png + def _deliver_type_text(self, text: str) -> None: + """Deliver a type step's text to the focused field. + + A native `` choice the backend delivered holds only while the + # option element it chose is still the select's one selected option, + # with the value and label captured before delivery. A page that + # replaced, removed or changed it is an unknown state: not landed, and + # nothing is retyped. + holds = getattr(self.backend, "native_select_choice_holds", None) + if callable(holds) and holds() is False: + return False, True # Masked input has no readable value: keep its pixel-only change # evidence local, so an unrelated change elsewhere cannot confirm it. region = self._field_region(field_point, field_region, masked=allow_masked) @@ -17842,7 +17873,11 @@ def _typed_input_landed( # Dropdown: typing selects the option whose label matches. When the # focused `` selection refuses the recording loudly: its option commit -happens in browser-native dropdown UI that produces no recordable action -events, so the choice would otherwise be silently absent from the compiled -workflow. A checkbox records only its click. ``contenteditable`` text entry -records the typed text. +native `` + + +
+
Choose a site
+
+ """ @@ -152,11 +191,412 @@ def drive(page, pump) -> None: assert type_event["text"] == "hello notes" -def test_native_select_refuses_the_recording(fixture_url: str, tmp_path: Path) -> None: +def _record_native_select( + fixture_url: str, out: Path, param_fields: tuple[str, ...] = () +) -> Path: + def drive(page, pump) -> None: + page.wait_for_selector("#sel", state="visible", timeout=20000) + kept = pump.__self__.events_kept + page.locator("#sel").click() + _settle(pump) + # On a slow runner the click may not be kept yet: pump until it is, + # so the type-ahead changes never share its batch (which the recorder + # refuses, by design). + for _ in range(200): + if pump.__self__.events_kept > kept: + break + pump() + assert pump.__self__.events_kept > kept, "the select click was never kept" + # Real key events, so the browser's own change event is trusted, as + # a person's selection is, at a person's pace: a change that arrives + # before the click's frame is kept is refused, by design. + page.wait_for_timeout(300) + kept = pump.__self__.events_kept + page.keyboard.type("Label Beta", delay=120) + # Linux Chromium opens a popup on the click; type-ahead only + # highlights an option there, and the change fires when the popup + # closes. Closed by the next click, the change and that click share + # one batch, which the recorder refuses, by design. Enter commits the + # choice first (and is no step: the change records it). Elsewhere the + # change has already fired and Enter does nothing (no form here). + page.keyboard.press("Enter") + _settle(pump) + # Pump until a change is kept and none has arrived for a settle + # window, as a person pauses before the next click. + quiet = 0 + for _ in range(400): + before = pump.__self__.events_kept + pump() + after = pump.__self__.events_kept + quiet = quiet + 1 if after == before and after > kept else 0 + if quiet >= 12: + break + assert pump.__self__.events_kept > kept, "the select change was never kept" + page.locator("#single").click() + _settle(pump) + + return record_interactive( + fixture_url, out, headless=True, script=drive, param_fields=param_fields + ) + + +def test_native_select_records_its_label(fixture_url: str, tmp_path: Path) -> None: + rec = _record_native_select(fixture_url, tmp_path / "rec") + events = _events(rec) + assert [e["kind"] for e in events] == ["click", "type", "click"] + assert events[1]["text"] == "Label Beta" + + workflow = compile_recording(rec, tmp_path / "bundle", name="select") + assert [step.action.value for step in workflow.steps] == ["click", "type", "click"] + + +def _compile_native_select(fixture_url: str, out: Path) -> Path: + """Record the select demonstration with ``sel`` as a parameter and + compile it; returns the bundle directory.""" + + rec = _record_native_select(fixture_url, out / "rec", ("sel",)) + bundle = out / "bundle" + workflow = compile_recording(rec, bundle, name="select") + typed = next(step for step in workflow.steps if step.action.value == "type") + assert typed.param == "sel" + return bundle + + +@pytest.fixture(scope="module") +def select_bundle(fixture_url: str, tmp_path_factory: pytest.TempPathFactory) -> Path: + return _compile_native_select(fixture_url, tmp_path_factory.mktemp("select")) + + +def _replay_native_select( + fixture_url: str, + tmp_path: Path, + value: str, + *, + bundle: Optional[Path] = None, + page_js: str = "", +) -> tuple[Any, Optional[str], int]: + """Replay the recorded select with ``value``: the report, the select's + value once the run ends, and how many key events reached the select. + ``page_js`` runs in the replay page before its own scripts (a handler + the app might have).""" + + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime import Replayer + + sync_api = pytest.importorskip("playwright.sync_api") + if bundle is None: + bundle = _compile_native_select(fixture_url, tmp_path) + with sync_api.sync_playwright() as pw: + browser = pw.chromium.launch(headless=True) + try: + page = browser.new_page(viewport={"width": 1280, "height": 800}) + page.add_init_script( + "window.__selKeys = 0; document.addEventListener('keydown'," + " (e) => { if (e.target.id === 'sel') window.__selKeys += 1; }," + " true);" + page_js + ) + page.goto(fixture_url) + page.wait_for_selector("#sel", state="visible") + report = Replayer(PlaywrightBackend(page)).run( + Workflow.load(bundle), + params={"sel": value}, + bundle_dir=bundle, + run_dir=tmp_path / "run", + ) + chosen = page.evaluate("document.getElementById('sel').value") + keys = page.evaluate("window.__selKeys") + finally: + browser.close() + return report, chosen, keys + + +def test_native_select_replay_chooses_the_labelled_option( + fixture_url: str, tmp_path: Path +) -> None: + # Recorded as Label Beta; replayed with another option. The option is + # chosen by its exact label (select_option), with no keystrokes, so a + # popup the click opened (Linux Chromium) commits nothing stale when the + # next click closes it. + report, chosen, keys = _replay_native_select(fixture_url, tmp_path, "Label Alpha") + assert report.success, [(r.step_id, r.ok, r.error) for r in report.results] + assert chosen == "v-alpha" + assert keys == 0 + + +def test_native_select_replay_refuses_a_missing_label( + fixture_url: str, tmp_path: Path +) -> None: + # "Label" names no option exactly (it only prefixes two). Replay refuses + # the step instead of typing, so type-ahead never lands on a near match, + # and the run stops before the next click. + report, chosen, keys = _replay_native_select(fixture_url, tmp_path, "Label") + assert not report.success + failed = [r for r in report.results if not r.ok] + assert len(failed) == 1 and failed[0].safety_halt + assert "labelled" in (failed[0].error or "") + assert report.results[-1] is failed[0] + assert chosen == "" + assert keys == 0 + + +# Handlers an app might attach to the select, run once when Label Alpha is +# chosen. Each but the last leaves an option other than the one replay chose. +_ON_ALPHA = ( + "document.addEventListener('DOMContentLoaded', () => {" + " const sel = document.getElementById('sel'); let done = false;" + " const act = () => {" + " if (done || sel.value !== 'v-alpha') return; done = true; %s };" + " sel.addEventListener('%s', () => { %s }); });" +) +_REPLACE = ( + "const old = sel.selectedOptions[0]; const twin = new Option(" + "'Label Alpha', 'v-shadow', true, true); sel.replaceChild(twin, old);" +) +_SELECT_HANDLERS = { + # Codex's case: the chosen option swapped for a same-label, same-index + # twin with another value. + "replace-on-change": _ON_ALPHA % (_REPLACE, "change", "act();"), + "replace-on-input": _ON_ALPHA % (_REPLACE, "input", "act();"), + "value-only": _ON_ALPHA + % ("sel.selectedOptions[0].value = 'v-shadow';", "change", "act();"), + "remove": _ON_ALPHA % ("sel.selectedOptions[0].remove();", "change", "act();"), + # After the events: only the read-back can see it. + "replace-later": _ON_ALPHA % (_REPLACE, "change", "setTimeout(act, 0);"), +} + + +@pytest.mark.parametrize("handler", sorted(_SELECT_HANDLERS)) +def test_native_select_replay_refuses_when_the_page_swaps_the_choice( + fixture_url: str, select_bundle: Path, tmp_path: Path, handler: str +) -> None: + report, _chosen, keys = _replay_native_select( + fixture_url, + tmp_path, + "Label Alpha", + bundle=select_bundle, + page_js=_SELECT_HANDLERS[handler], + ) + assert not report.success + failed = [r for r in report.results if not r.ok] + assert len(failed) == 1, [(r.step_id, r.ok, r.error) for r in report.results] + assert failed[0].input_verified is False + assert failed[0].delivery_attempted is True + # The run halted on the select: the click after it never ran. + assert report.results[-1] is failed[0] + assert keys == 0 + + +def test_native_select_replay_verifies_with_a_benign_handler( + fixture_url: str, select_bundle: Path, tmp_path: Path +) -> None: + # A handler that touches unrelated DOM leaves the choice as chosen. + benign = _ON_ALPHA % ( + "document.body.appendChild(document.createElement('div'))" + ".textContent = 'saved';", + "change", + "act();", + ) + report, chosen, keys = _replay_native_select( + fixture_url, tmp_path, "Label Alpha", bundle=select_bundle, page_js=benign + ) + assert report.success, [(r.step_id, r.ok, r.error) for r in report.results] + typed = next(r for r in report.results if r.input_verified is not None) + assert typed.input_verified is True + assert chosen == "v-alpha" + assert keys == 0 + + +def _choose_in_select(options_html: str, label: str) -> tuple[Any, str, int, list[str]]: + """Focus ``#s`` built from ``options_html`` and ask the backend to choose + ``label``: (the chooser's result or the exception it raised, the select's + value afterwards, how many change events fired, each option's label as + the recorder stores it).""" + + from openadapt_flow.backends.playwright_backend import ( + NATIVE_OPTION_LABEL_JS, + PlaywrightBackend, + ) + + sync_api = pytest.importorskip("playwright.sync_api") + with sync_api.sync_playwright() as pw: + browser = pw.chromium.launch(headless=True) + try: + page = browser.new_page() + page.set_content( + '" + ) + stored = page.evaluate( + "Array.from(document.getElementById('s').options).slice(1)" + f".map({NATIVE_OPTION_LABEL_JS})" + ) + page.focus("#s") + outcome: Any + try: + outcome = PlaywrightBackend(page).select_focused_native_option(label) + except Exception as exc: # noqa: BLE001 - the refusal is the result + outcome = exc + value = page.evaluate("document.getElementById('s').value") + changes = page.evaluate("window.__changes") + finally: + browser.close() + return outcome, value, changes, stored + + +def _assert_refused(outcome: Any, value: str, changes: int) -> None: + from openadapt_flow.backend import StructuralResolutionRefused + + assert isinstance(outcome, StructuralResolutionRefused), outcome + assert value == "" + assert changes == 0 + + +_EXACT = '' + + +@pytest.mark.parametrize( + "decoy", + [ + '', + '', + ], + ids=["text", "attr"], +) +def test_native_select_choice_refuses_a_whitespace_decoy(decoy: str) -> None: + # Recording and replay squash whitespace alike, so "Label Alpha" and + # "Label Alpha" are one label: two matches, refused (the recorder refuses + # such a select too). Never a wrong-option success. + outcome, value, changes, stored = _choose_in_select(decoy + _EXACT, "Label Alpha") + assert stored == ["Label Alpha", "Label Alpha"] + _assert_refused(outcome, value, changes) + + +def test_native_select_choice_refuses_duplicates() -> None: + outcome, value, changes, _stored = _choose_in_select( + '' + '', + "Label Alpha", + ) + _assert_refused(outcome, value, changes) + + +def test_native_select_choice_refuses_a_disabled_match() -> None: + outcome, value, changes, _stored = _choose_in_select( + '', "Label Alpha" + ) + _assert_refused(outcome, value, changes) + + +def test_native_select_choice_selects_a_multiline_option() -> None: + outcome, value, changes, stored = _choose_in_select( + '' + '', + "Dog", + ) + assert stored == ["Dog", "Cat"] + assert outcome is True + assert value == "v-dog" + assert changes == 1 + + +# One case per whitespace class, as the recorder stores it (JavaScript \s, +# squashed and trimmed) and replay therefore matches it. +@pytest.mark.parametrize( + ("option_html", "stored_label"), + [ + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + ('', "Label Alpha"), + # Zero-width space is not \s: it stays, so "Label Alpha" is no match. + ('', "Label\u200b Alpha"), + # Case is kept: the recorder stores it as shown. + ('', "label alpha"), + ], + ids=[ + "double-space", + "tab", + "newline", + "leading-trailing", + "nbsp", + "em-space", + "label-attr", + "zero-width-space", + "case", + ], +) +def test_native_select_choice_follows_the_recorders_whitespace( + option_html: str, stored_label: str +) -> None: + outcome, value, changes, stored = _choose_in_select(option_html, "Label Alpha") + assert stored == [stored_label] + if stored_label == "Label Alpha": + assert outcome is True + assert value == "v" + assert changes == 1 + else: + _assert_refused(outcome, value, changes) + + +def test_native_select_with_a_shared_label_is_refused( + fixture_url: str, tmp_path: Path +) -> None: + # Two options read "Main clinic" (case aside): replay can only choose by + # label, so it would pick the first. The choice is refused, not recorded. + def drive(page, pump) -> None: + page.wait_for_selector("#dup", state="visible", timeout=20000) + page.locator("#dup").click() + _settle(pump) + page.keyboard.press("ArrowDown") + page.keyboard.press("ArrowDown") + _settle(pump) + + try: + rec = record_interactive( + fixture_url, tmp_path / "rec", headless=True, script=drive + ) + except Exception as exc: # the recorder may stop on the refusal + assert "select" in str(exc).lower() + return + assert not [e for e in _events(rec) if e["kind"] == "type"] + + +def test_widget_over_a_hidden_select_records_its_clicks( + fixture_url: str, tmp_path: Path +) -> None: + def drive(page, pump) -> None: + page.wait_for_selector("#site-control", state="visible", timeout=20000) + page.locator("#site-control").click() + _settle(pump) + page.get_by_role("option", name="Site Two").click() + _settle(pump) + assert page.evaluate("document.getElementById('site').value") == "2" + + rec = record_interactive(fixture_url, tmp_path / "rec", headless=True, script=drive) + events = _events(rec) + assert [e["kind"] for e in events] == ["click", "click"] + assert events[1]["structural"]["name"] == "Site Two" + + +def test_script_set_select_value_is_not_recorded( + fixture_url: str, tmp_path: Path +) -> None: + """Playwright's select_option sets the value from script, as a widget + does: nothing is recorded for it, and the recording is not refused.""" + def drive(page, pump) -> None: page.wait_for_selector("#sel", state="visible", timeout=20000) page.locator("#sel").select_option(label="Label Alpha") _settle(pump) + page.locator("#single").click() + _settle(pump) - with pytest.raises(BrowserAttachError, match="select"): - record_interactive(fixture_url, tmp_path / "rec", headless=True, script=drive) + rec = record_interactive(fixture_url, tmp_path / "rec", headless=True, script=drive) + assert [e["kind"] for e in _events(rec)] == ["click"] diff --git a/packages/flow/tests/test_page_settle.py b/packages/flow/tests/test_page_settle.py index cd3c777c3..55ba71c4a 100644 --- a/packages/flow/tests/test_page_settle.py +++ b/packages/flow/tests/test_page_settle.py @@ -377,13 +377,21 @@ def test_dropdown_typed_input_verified_from_dom() -> None: assert landed and changed and ocr_calls == 0 -@pytest.mark.parametrize("label", [None, "Maine", "Massachusetts (MA)"]) -def test_dropdown_dom_miss_falls_back_and_never_passes(label) -> None: - (landed, _changed), ocr_calls = _typed(label) - assert landed is False # OCR (empty here) decides; DOM miss never passes +def test_dropdown_dom_miss_falls_back_and_never_passes() -> None: + (landed, _changed), ocr_calls = _typed(None) + assert landed is False # no focused select: OCR (empty here) decides assert ocr_calls >= 1 +@pytest.mark.parametrize("label", ["Maine", "Massachusetts (MA)"]) +def test_dropdown_showing_another_option_fails_without_ocr(label) -> None: + # The focused select shows another option (type-ahead stopped elsewhere): + # the read-back decides, and OCR can't turn it into a pass. + (landed, changed), ocr_calls = _typed(label) + assert landed is False and changed is True + assert ocr_calls == 0 + + def _seed_from(result): return settle_mod.SettleSeed( phash=result.phash, diff --git a/packages/flow/tests/test_structural_locator_idless.py b/packages/flow/tests/test_structural_locator_idless.py new file mode 100644 index 000000000..6d0b91348 --- /dev/null +++ b/packages/flow/tests/test_structural_locator_idless.py @@ -0,0 +1,204 @@ +"""Id-less controls: which get a structural selector, and which never do. + +A field with no ``id`` but a unique ``name`` (Trac's ``field_summary``) gets +``input[name=...]``. A button never does: its ``name`` is the form field it +submits (Trac's ````, NetBox's +`` + + + + + +""" + + +@pytest.fixture(scope="module") +def page_url(tmp_path_factory: pytest.TempPathFactory) -> str: + path = tmp_path_factory.mktemp("idless") / "newticket.html" + path.write_text(PAGE) + return path.as_uri() + + +@pytest.fixture(scope="module") +def browser() -> Iterator[Any]: + with sync_api.sync_playwright() as playwright: + launched = playwright.chromium.launch(headless=True) + try: + yield launched + finally: + launched.close() + + +def _center(page: Any, selector: str) -> tuple[int, int]: + box = page.locator(selector).first.bounding_box() + assert box is not None + return int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + + +def test_locator_names_idless_buttons_by_their_label( + browser: Any, page_url: str +) -> None: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + page.goto(page_url) + backend = PlaywrightBackend(page) + + trac = backend.structural_locator_at(*_center(page, "input[name=submit]")) + assert trac is not None + # A button is named by its own label, never by the field name it + # submits: `name` alone could find another button at replay. + assert trac.selector is None + assert (trac.role, trac.name) == ("button", "Create ticket") + + netbox = backend.structural_locator_at(*_center(page, "button[name=_create]")) + assert netbox is not None + assert netbox.selector is None + assert (netbox.role, netbox.name) == ("button", "Create") + + summary = backend.structural_locator_at( + *_center(page, "input[name=field_summary]") + ) + assert summary is not None + assert summary.selector == 'input[name="field_summary"]' + + # Two identical unnamed buttons: no selector that could pick the + # wrong one, only role and name. + twin = backend.structural_locator_at(*_center(page, "button:text('Save')")) + assert twin is not None + assert twin.selector is None + assert (twin.role, twin.name) == ("button", "Save") + finally: + page.close() + + +def test_recorded_create_step_carries_its_selector( + browser: Any, page_url: str, tmp_path: Path +) -> None: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + page.goto(page_url) + recorder = Recorder(PlaywrightBackend(page), tmp_path / "rec", app_url=page_url) + recorder.click(*_center(page, "input[name=field_summary]")) + recorder.type_text("Printer jams on tray 2", param="summary") + recorder.click(*_center(page, "input[name=submit]")) + recorder.finish() + assert page.evaluate("window.actions") == ["submit"] + finally: + page.close() + + workflow = compile_recording(tmp_path / "rec", tmp_path / "bundle", name="ticket") + save = workflow.steps[-1] + assert save.anchor is not None and save.anchor.structural is not None + assert save.anchor.structural.selector is None + assert save.anchor.structural.role == "button" + assert save.anchor.structural.name == "Create ticket" + + +SHADOW_PAGE = """ + +

+ + +""" + + +def test_a_lookalike_in_another_root_never_names_the_clicked_button( + browser: Any, tmp_path: Path +) -> None: + """A Save inside a shadow root, and a look-alike Save in the page with + the same name. ``button[data-testid="save"]`` matches exactly one element in the + document, the look-alike, so a count alone would call it unique for the + shadow button and a replay would click the wrong Save.""" + + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + path = tmp_path / "shadow.html" + path.write_text(SHADOW_PAGE) + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + page.goto(path.as_uri()) + backend = PlaywrightBackend(page) + shadow = page.locator("save-panel").locator("button") + box = shadow.bounding_box() + assert box is not None + x, y = int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + + # The page's own hit test stops at the shadow host. + located = backend.structural_locator_at(x, y) + assert located is None or located.selector != 'button[data-testid="save"]' + + # A hit test that reaches the button inside the shadow root (as a + # piercing one would): still no selector, because the only match in + # the document is the look-alike. + page.evaluate( + """() => { + const inner = document.querySelector('save-panel') + .shadowRoot.querySelector('button'); + document.elementFromPoint = () => inner; + }""" + ) + located = backend.structural_locator_at(x, y) + assert located is not None + assert located.selector is None + assert (located.role, located.name) == ("button", "Save") + + # The light-DOM Save is the one element its selector matches, and it + # is the clicked one: it keeps the selector. + page.reload() + light = page.locator("p > button") + box = light.bounding_box() + assert box is not None + located = backend.structural_locator_at( + int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + ) + assert located is not None and located.selector == 'button[data-testid="save"]' + finally: + page.close()