fix: record native and widget selects, and give id-less save buttons a selector

packages/flow/docs/BROWSER_RECORDING.md

@@ -397,12 +397,26 @@ gesture compiles to one `DOUBLE_CLICK` step), right click, drag, text entry
 (including `contenteditable` and `role="textbox"`), special keys, keyboard
 shortcuts, and wheel scrolling.
 
+A native `<select>` choice, by mouse or by typing, is recorded as the chosen
+option's label typed into the select (whitespace runs made one space, case
+kept). Replay compares each option's label squashed the same way, chooses
+the one option that matches, with no keystrokes, and reads the select's
+chosen option back. No match, or two options that only differ in
+whitespace, stops the run before anything is chosen; a select that ends on
+another option stops it too. Parameterize it like any
+field (`--param` with the select's `name`) to choose a different option on
+each run; a value that is no option's exact label stops the run before
+anything is chosen. No Enter is recorded for it. A choice made from a widget drawn over a
+hidden select (TomSelect, Select2) is recorded as the clicks and typing on the
+widget; the change the widget's script then sends to the hidden select is not
+recorded again. A multiple-choice select is refused.
+
 Refused loudly, never recorded silently wrong:
 
-- A native `<select>` selection. Its option commit happens in browser-native
-  dropdown UI that produces no recordable action events, so the choice would
-  be silently absent from the compiled workflow. The recording stops with an
-  explicit error at the moment of selection.
+- A multiple-choice `<select multiple>`, a select that holds a declared
+  secret, or a choice whose label another option of the select also shows
+  (case aside): replay chooses by label and couldn't tell them apart. The
+  recording stops at the moment of selection.
 - In attach mode, a new tab that no followed page opened, or a popup on
   another origin. Popups and tabs that a followed page opens are recorded as
   surfaces ([MULTI_SURFACE.md](MULTI_SURFACE.md)). A popup that closes itself

packages/flow/openadapt_flow/backend.py

@@ -44,6 +44,16 @@ class StructuralElementNotActionable(StructuralResolutionRefused):
     """
 
 
+class StructuralRefusalAfterDelivery(StructuralResolutionRefused):
+    """A safety refusal raised after the backend already delivered input.
+
+    The backend acted and then found the result is not exactly what the step
+    asked for (a native select that did not keep the exactly labelled
+    option). It halts like any :class:`StructuralResolutionRefused`, but the
+    step's delivery is reported as attempted: an input edge was crossed.
+    """
+
+
 class ActionDeliveryUncertain(RuntimeError):
     """An action API failed after delivery may already have begun.
 

packages/flow/openadapt_flow/backends/playwright_backend.py

@@ -31,6 +31,7 @@
 from openadapt_flow.backend import (
     ActionDeliveryUncertain,
     StructuralElementNotActionable,
+    StructuralRefusalAfterDelivery,
     StructuralResolutionRefused,
 )
 from openadapt_flow.ir import (
@@ -1157,6 +1158,113 @@ def _fill_iso_date(target: Any, text: str) -> bool:
     return True
 
 
+#: A native option's label as the interactive recorder stores it and as
+#: replay matches it: ``option.label`` (the label attribute, else the text),
+#: every run of JavaScript whitespace (``\s``: tab, newline, no-break space
+#: and the other Unicode spaces) made one space, then trimmed. Case is kept.
+#: One function for both, so a recorded label and the option it names agree.
+NATIVE_OPTION_LABEL_JS = (
+    r"(o) => String(o.label || o.text || '').replace(/\s+/g, ' ').trim()"
+)
+
+#: Choose, in a native single ``<select>``, the one option whose label
+#: (``NATIVE_OPTION_LABEL_JS``) is exactly ``label``, and fire ``input`` and
+#: ``change`` as ``select_option`` does. Returns a record: ``{kind: 'none'}``
+#: when the target is no native select; ``{kind: 'refused'}`` (nothing
+#: changed) for a multiple or disabled select, zero or several matches, or a
+#: disabled match; else ``{kind: 'selected'}`` with the chosen option ELEMENT,
+#: its value and label captured BEFORE the events ran (a page handler cannot
+#: reach this record), and ``held``: the choice check right after them.
+_SELECT_EXACT_LABEL_JS = (
+    r"""(e, label) => {
+    if (e.tagName !== 'SELECT') return {kind: 'none'};
+    if (e.multiple || e.disabled) return {kind: 'refused'};
+    const optionLabel = """
+    + NATIVE_OPTION_LABEL_JS
+    + r""";
+    const matches = Array.from(e.options).filter((o) => optionLabel(o) === label);
+    if (matches.length !== 1 || matches[0].disabled) return {kind: 'refused'};
+    const option = matches[0];
+    const record = {
+        kind: 'selected', select: e, option: option,
+        value: option.value, label: optionLabel(option),
+    };
+    option.selected = true;
+    e.dispatchEvent(new Event('input', {bubbles: true, composed: true}));
+    e.dispatchEvent(new Event('change', {bubbles: true}));
+    record.held = (__HOLDS__)(record);
+    return record;
+}"""
+)
+
+#: Does a choice record still hold? The same option element (not one at the
+#: same index or with the same label) is connected, inside the select, its
+#: only selected option, with the value and label captured before delivery.
+_NATIVE_CHOICE_HOLDS_JS = (
+    r"""(r) => {
+    const optionLabel = """
+    + NATIVE_OPTION_LABEL_JS
+    + r""";
+    const e = r.select, o = r.option;
+    const selected = e.selectedOptions;
+    return e.isConnected && o.isConnected && e.contains(o)
+        && selected.length === 1 && selected[0] === o
+        && o.value === r.value && optionLabel(o) === r.label;
+}"""
+)
+_SELECT_EXACT_LABEL_JS = _SELECT_EXACT_LABEL_JS.replace(
+    "__HOLDS__", _NATIVE_CHOICE_HOLDS_JS
+)
+
+
+def _select_native_option(target: Any, text: str) -> Optional[Any]:
+    """Choose the option labelled exactly ``text`` in a native ``<select>``.
+
+    Typing into a select only highlights an option where the browser opens a
+    popup for it (Linux Chromium), and commits nothing until the popup
+    closes; Enter to commit could submit a form. The option is chosen on the
+    element itself, firing ``input`` and ``change`` with no key events. The
+    match is on the label as the recorder stores it
+    (``NATIVE_OPTION_LABEL_JS``), so a multi-line ``<option>`` recorded as
+    "Dog" matches, and an option that only differs in whitespace from another
+    ("Label  Alpha" beside "Label Alpha") makes two matches. ``target`` is an
+    element handle.
+
+    Returns ``None``, with nothing delivered, when the target is no native
+    select. Once it is, zero or several matches, a disabled match, or a
+    multiple or disabled select are refused with nothing delivered: replay
+    never falls back to typing. The chosen option element, value and label
+    are captured in the page before the events run; the expected value is
+    never read from the page after them. Unless that same element is still
+    connected and the select's only selected option, with the captured value
+    and the requested label (checked right after the events and again
+    here), :class:`StructuralRefusalAfterDelivery` is raised. Otherwise
+    returns the choice record's handle, which ``_NATIVE_CHOICE_HOLDS_JS``
+    rechecks at the caller's read-back.
+    """
+
+    record = target.evaluate_handle(_SELECT_EXACT_LABEL_JS, text)
+    meta = record.evaluate("r => ({kind: r.kind, label: r.label, held: r.held})")
+    kind = meta.get("kind") if isinstance(meta, dict) else None
+    if kind == "none":
+        return None
+    if kind != "selected":
+        raise StructuralResolutionRefused(
+            "the focused native <select> has no single enabled option labelled "
+            "exactly as the step's value; refusing to choose one"
+        )
+    if not (
+        meta.get("label") == text
+        and meta.get("held") is True
+        and record.evaluate(_NATIVE_CHOICE_HOLDS_JS) is True
+    ):
+        raise StructuralRefusalAfterDelivery(
+            "the native <select> did not keep the option chosen for the step's "
+            "value (the page replaced, removed or changed it)"
+        )
+    return record
+
+
 _TEXT_AT_POINT_JS = r"""([px, py, target]) => {
     // Recorded texts come from OCR, which drops spaces and misreads
     // punctuation ("October-2026-" for "October 2026"): keep letters and
@@ -1887,22 +1995,51 @@ def _fill_iso_date(target: Any, text: str) -> bool:
     ) || el;
     const tag = actionable.tagName.toLowerCase();
     let selector = null;
+    // A selector counts only when, in the clicked element's own document,
+    // it matches exactly one element and that element is the clicked one. A
+    // look-alike elsewhere must not make a selector "unique" for the wrong
+    // element. Inside a shadow root no selector is built: Playwright's CSS
+    // locator pierces open shadow roots at replay, so a selector unique in
+    // one root could match a look-alike in the page or another root.
+    const root = actionable.getRootNode();
+    const unique = (candidate) => {
+        if (!(root instanceof Document)) return false;
+        try {
+            return root.querySelectorAll(candidate).length === 1 &&
+                root.querySelector(candidate) === actionable;
+        } catch (_) { return false; }
+    };
     const id = actionable.id;
-    if (id && document.querySelectorAll(
-            '#' + CSS.escape(id)).length === 1) {
+    if (id && unique('#' + CSS.escape(id))) {
         selector = '#' + CSS.escape(id);
     }
     // <input type=submit> and its kin are buttons named by their value,
     // as the accessibility tree (and get_by_role) names them.
     const inputType = (actionable.getAttribute('type') || '').toLowerCase();
     const inputButton = tag === 'input' &&
         ['submit', 'button', 'reset', 'image'].includes(inputType);
+    // No unique id: the same unique attributes the interactive recorder uses.
+    // A button's `name` alone names the form field it submits, not the
+    // button: `button[name=action]` would find Deny where Approve was
+    // recorded, so a button keeps its role and label. A bracketed name
+    // (`items[0][qty]`) is a row position, which a reordered list moves to
+    // another row. Neither is a selector.
+    const buttonLike = inputButton || tag === 'button';
+    for (const attr of ['data-testid', 'data-test', 'name']) {
+        if (selector) break;
+        const value = actionable.getAttribute(attr);
+        if (!value) continue;
+        if (attr === 'name' && (buttonLike || /\[/.test(value))) continue;
+        const candidate = tag + '[' + attr + '="' + CSS.escape(value) + '"]';
+        if (unique(candidate)) selector = candidate;
+    }
     let role = actionable.getAttribute('role');
     if (!role) {
         const map = {button: 'button', a: 'link', input: 'textbox',
             select: 'combobox', textarea: 'textbox'};
         role = inputButton ? 'button' : (map[tag] || null);
         if (tag === 'a' && !actionable.getAttribute('href')) role = null;
+        if (inputType === 'checkbox' || inputType === 'radio') role = inputType;
     }
     const fieldLabel = """
     + _FIELD_LABEL_JS
@@ -2213,6 +2350,10 @@ def __init__(
         # references only) for submit bindings. A new backend starts empty.
         self._fill_store_key = f"__oaflow_filled_{uuid.uuid4().hex}"
         self._fill_value_slot = f"__oaflow_filled_values_{uuid.uuid4().hex}"
+        # The last native <select> choice this backend delivered (a page-side
+        # record of the option element and its pre-delivery value), for the
+        # replayer's read-back; cleared by every text delivery.
+        self._native_select_choice: Optional[Any] = None
         self._form_wire: Any = None
         # The page-side guard (see _arm_wire_check): one per browser context,
         # kept in each document under a private key, installed by an init
@@ -4906,6 +5047,11 @@ def _act_guarded_keyboard(
             self._assert_qualification_environment_current()
             try:
                 deliver(token_locator)
+            except StructuralResolutionRefused:
+                # A typed refusal from the delivery itself (a native select
+                # with no exactly labelled option) says what was delivered:
+                # nothing, or, as StructuralRefusalAfterDelivery, something.
+                raise
             except Exception as exc:
                 raise ActionDeliveryUncertain(
                     operation=operation,
@@ -4954,6 +5100,7 @@ def type_text_guarded(
     ) -> ActionDeliveryReceipt:
         """Type through the pre-identity focused-element lease."""
 
+        self._native_select_choice = None
         self._register_focused_field()
         receipt = self._act_guarded_keyboard(
             expected_frame_sha256=expected_frame_sha256,
@@ -4965,6 +5112,9 @@ def type_text_guarded(
                     _is_iso_date(text)
                     and _fill_iso_date(locator.element_handle(timeout=1000), text)
                 )
+                or self._choose_native_option(
+                    locator.element_handle(timeout=1000), text
+                )
                 or locator.press_sequentially(text, timeout=1000)
             ),
         )
@@ -5554,6 +5704,62 @@ def _fill_focused_iso_date(self, text: str) -> bool:
             return False
         return target is not None and _fill_iso_date(target, text)
 
+    def select_focused_native_option(self, text: str) -> bool:
+        """Choose ``text`` by its exact label when the focused element is a
+        native ``<select>``, with no keystrokes; ``False`` (nothing delivered)
+        when it is not one. A label no single option carries raises
+        :class:`StructuralResolutionRefused` (see
+        :func:`_select_native_option`). The replayer calls this for a type
+        step before typing; recording and other callers keep ``type_text``.
+        """
+
+        self._native_select_choice = None
+        self._assert_qualification_environment_current()
+        try:
+            target = self._focused_element_handle()
+            is_select = target is not None and target.evaluate(
+                "e => e.tagName === 'SELECT'"
+            )
+        except Exception:  # noqa: BLE001 - unreadable focus: not a select
+            return False
+        if is_select is not True:
+            return False
+        # As type_text does: submit bindings and create-form identity checks
+        # see the select as a field this program filled.
+        self._register_focused_field()
+        self._choose_native_option(target, text)
+        self._snapshot_focused_field(text)
+        return True
+
+    def _choose_native_option(self, target: Any, text: str) -> bool:
+        """:func:`_select_native_option`, keeping its choice record for
+        :meth:`native_select_choice_holds`."""
+
+        self._native_select_choice = None
+        record = _select_native_option(target, text)
+        if record is None:
+            return False
+        self._native_select_choice = record
+        return True
+
+    def native_select_choice_holds(self) -> Optional[bool]:
+        """Does the last native ``<select>`` choice still hold?
+
+        ``None`` when the last text delivery chose no native option. Else
+        ``True`` only while the option element chosen then is connected, the
+        select's only selected option, with the value and label captured
+        before delivery; an unreadable record is ``False``. The expected value
+        is never read from the page after delivery.
+        """
+
+        record = self._native_select_choice
+        if record is None:
+            return None
+        try:
+            return record.evaluate(_NATIVE_CHOICE_HOLDS_JS) is True
+        except Exception:  # noqa: BLE001 - unreadable after delivery: not held
+            return False
+
     def _deliver_text(self, text: str, per_key: Callable[[], None]) -> None:
         """Insert ``text`` at the caret in one call when the focused element
         is an ordinary text field (native fill: the field's ``input`` event
@@ -5569,6 +5775,7 @@ def _deliver_text(self, text: str, per_key: Callable[[], None]) -> None:
 
     def type_text(self, text: str) -> None:
         """Type text into the currently focused element."""
+        self._native_select_choice = None
         self._assert_qualification_environment_current()
         self._register_focused_field()
         self._deliver_text(text, lambda: self.page.keyboard.type(text))