diff --git a/CAPABILITIES.md b/CAPABILITIES.md
index e7e2c5054a..6c31f80b8d 100644
--- a/CAPABILITIES.md
+++ b/CAPABILITIES.md
@@ -145,6 +145,7 @@ improvising or retrying when something is off.
| Compile one or more demonstrations into a deterministic program | Released | Real app (OpenEMR referral, 62 timed runs) | `openadapt-flow record`, `compile`; [packages/flow](packages/flow/README.md) |
| Classify common local edits without a person (select-all, Enter in a field or dropdown, an in-form tab, an informational dialog's dismiss), only when the recording's network capture shows no write and a runtime check confirms only the expected change | On main ([#1366](https://github.com/OpenAdaptAI/OpenAdapt/pull/1366)) | Synthetic; OpenEMR referral recording re-admitted offline (violations 12 to 1 under `permissive`) | [LOCAL_EDIT_CLASSIFICATION.md](packages/flow/docs/design/LOCAL_EDIT_CLASSIFICATION.md), `compiler/local_edit.py` |
| Keep the empty search result a person confirmed through (a duplicate check's "No matches were found.") as an exact check after the click that opens the dialog and again right before Confirm, so a run whose dialog lists a match stops before the save; only fixed-vocabulary search-result statements shown once, never text with a name or search term, and a confirmation dialog without one is noted in `compile_notes.json` | On main | Real app frames (an OpenEMR referral recording, compiled offline) | `compiler/empty_result_guard.py` |
+| Mine that check in any language: induction keeps the one line a named dialog adds (not its title or buttons, not a question, no typed value or number) when two or more demonstrations whose typed values all differ show the same line, refuses when the demonstrations disagree on a mined check instead of copying the first one's, and qualification then needs a business-rule run that halts at it | On main | Synthetic (a search dialog recorded in Chromium by the real recorder, German statement, real OCR) | `compiler/empty_result_guard.py`, `compiler/induction.py` |
| Qualify against a rule the app itself enforces: verify.yaml `business_rules` seed the conflicting record (an existing chart for the same person, checked in the read-back), and the certified bundle must stop at the rule's check before the save with nothing written on every trial, or certification is withdrawn | On main | Unit tests; the OpenEMR fixture's verify.yaml declares the existing-duplicate rule | [QUALIFY_A_WORKFLOW.md](packages/flow/docs/QUALIFY_A_WORKFLOW.md) |
| Right-record check before any write (structure, then OCR, then pixels) | Released | Real app and synthetic | [RESOLUTION_LADDER.md](packages/flow/docs/RESOLUTION_LADDER.md), `runtime/identity.py` |
| Read the saved record back from the system of record (API, database, FHIR, separate session) | Released | Real app (OpenEMR 12 of 12 fields) | [VERIFICATION.md](packages/flow/docs/VERIFICATION.md), `runtime/effects/` |
diff --git a/llms-full.txt b/llms-full.txt
index 908d009aac..643aa6048a 100644
--- a/llms-full.txt
+++ b/llms-full.txt
@@ -548,6 +548,7 @@ improvising or retrying when something is off.
| Compile one or more demonstrations into a deterministic program | Released | Real app (OpenEMR referral, 62 timed runs) | `openadapt-flow record`, `compile`; [packages/flow](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/README.md) |
| Classify common local edits without a person (select-all, Enter in a field or dropdown, an in-form tab, an informational dialog's dismiss), only when the recording's network capture shows no write and a runtime check confirms only the expected change | On main ([#1366](https://github.com/OpenAdaptAI/OpenAdapt/pull/1366)) | Synthetic; OpenEMR referral recording re-admitted offline (violations 12 to 1 under `permissive`) | [LOCAL_EDIT_CLASSIFICATION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/design/LOCAL_EDIT_CLASSIFICATION.md), `compiler/local_edit.py` |
| Keep the empty search result a person confirmed through (a duplicate check's "No matches were found.") as an exact check after the click that opens the dialog and again right before Confirm, so a run whose dialog lists a match stops before the save; only fixed-vocabulary search-result statements shown once, never text with a name or search term, and a confirmation dialog without one is noted in `compile_notes.json` | On main | Real app frames (an OpenEMR referral recording, compiled offline) | `compiler/empty_result_guard.py` |
+| Mine that check in any language: induction keeps the one line a named dialog adds (not its title or buttons, not a question, no typed value or number) when two or more demonstrations whose typed values all differ show the same line, refuses when the demonstrations disagree on a mined check instead of copying the first one's, and qualification then needs a business-rule run that halts at it | On main | Synthetic (a search dialog recorded in Chromium by the real recorder, German statement, real OCR) | `compiler/empty_result_guard.py`, `compiler/induction.py` |
| Qualify against a rule the app itself enforces: verify.yaml `business_rules` seed the conflicting record (an existing chart for the same person, checked in the read-back), and the certified bundle must stop at the rule's check before the save with nothing written on every trial, or certification is withdrawn | On main | Unit tests; the OpenEMR fixture's verify.yaml declares the existing-duplicate rule | [QUALIFY_A_WORKFLOW.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/QUALIFY_A_WORKFLOW.md) |
| Right-record check before any write (structure, then OCR, then pixels) | Released | Real app and synthetic | [RESOLUTION_LADDER.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/RESOLUTION_LADDER.md), `runtime/identity.py` |
| Read the saved record back from the system of record (API, database, FHIR, separate session) | Released | Real app (OpenEMR 12 of 12 fields) | [VERIFICATION.md](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/VERIFICATION.md), `runtime/effects/` |
@@ -1251,7 +1252,7 @@ signed; the Python packages are published from the reviewed release workflow.
| Deterministic re-resolution | **Required CI** | Bounded visual or structural drift can be resolved through non-model evidence and recorded as a reviewable change. | It is not general adaptation to a redesigned workflow, changed business rules, or missing evidence. |
| AI-assisted repair | **Required CI contracts; measured on synthetic apps; deployment evidence required** | An explicitly enabled model can propose a target or interpret a changed screen. Existing runtime checks still apply. Once a deployment names a repair agent, every stop that wrote nothing goes to it by default, and a person approves each fix unless a signed policy pre-approves its class. On MockMed's 22 drift modes, 6 of 21 stops that reached repair resumed and verified, with 0 wrong saves in 66 trials ([benchmark/auto_repair](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/../benchmark/auto_repair/README.md)). | A model proposal is not authorization, proof of identity, or proof that a business transaction succeeded. A stop after a possible write is never repaired automatically. The measured rate is for one agent on one synthetic app; it is not a field repair rate. |
| Human teaching and resume | **Required CI contracts; field evidence required** | A halt can produce evidence for an operator correction, guarded promotion, and durable resume. | Field recovery time, broad authoring UX, enterprise identity integration, and non-repudiation are not established. |
-| Windows UI Automation | **Counted task acceptance** | The native UIA backend completed 3/3 exact effects in an in-tree WinForms app in a Windows 11 VM, and 3/3 refusals for both stale and ambiguous targets. | Acceptance is bound to the tested workflow and Windows environment; each application retains its own identity, effect, and version qualification. |
+| Windows UI Automation | **Counted task acceptance** | The native UIA backend completed 3/3 exact effects in an in-tree WinForms app in a Windows 11 VM, and 3/3 refusals for both stale and ambiguous targets. | Acceptance is bound to the tested workflow and Windows environment; each application retains its own identity, effect, and version qualification. A qualification case pins the foreground executable's file digest, the machine and its exact OS build, so an OS update or a new executable needs a new qualification. For an app that a script interpreter, browser or shared runtime hosts, that digest pins the host executable, so the app's own release is pinned by requalifying it. |
| Native macOS automation | **Counted task acceptance** | The AX/AppKit backend, driven directly, completed 3/3 exact TextEdit file effects and refused an ambiguous two-window target; structural record/locate/refuse behavior is covered separately. Known AX-only ` - Google Chrome` and ` - Google Chrome for Testing` window-title suffixes are accepted only for the corresponding exact CoreGraphics application owner; PID, unique-candidate, focused/main, and topmost-window checks remain mandatory. | Acceptance is bound to the tested host and application. Accessibility exposure and supported native actions vary by application. |
| Native Linux automation | **Unit tests plus CI recording** | Root CI records under Xvfb and reads a GTK3 button over AT-SPI. Replay and the stops on ambiguous and stale targets are unit-tested with a fake AT-SPI client. | The built-in path is X11. Wayland needs an operator-approved portal session, and each application retains its own qualification. |
| RDP automation | **Counted task acceptance** | An Aardwolf batch into a Windows 11 VM passed 3/3 (transport and input) with an independent guest-tools oracle; a separate real-FreeRDP batch covers record, compile, governed replay, and drift refusal. | The records do not cover arbitrary applications, WAN conditions, or Citrix ICA/HDX. A real AVC444 session (same virtual network, software encoding) stops on 12 of 20 healthy OpenEMR runs; the causes are ranked in benchmark/rdp_avc_real. |
diff --git a/packages/flow/CHANGELOG.md b/packages/flow/CHANGELOG.md
index fc400a3d1b..c1ebbc135f 100644
--- a/packages/flow/CHANGELOG.md
+++ b/packages/flow/CHANGELOG.md
@@ -27,6 +27,8 @@
**Breaking:** admission of every JIT draft now needs one (`teach_auth_not_recorded` otherwise), in every admission mode; so does a self-repair of an agent-taught program and the approval of a calling agent's fix. Drafts from the airbag handoff are refused, and browser teach (`serve --teach-browser`) no longer teaches or runs write tools.
+- A dialog's empty-result check in any language. Induction mines the one line a named confirmation dialog adds when every demonstration shows the same line and every value typed before the dialog differs between them. Questions, numbers, typed values and lines from one demonstration only are not mined. When the demonstrations disagree on a mined check, induction refuses (`ambiguous_guard`) instead of copying the first recording's. The opener and its Confirm click no longer read as a two-row loop. Qualification asks for a business-rule case that halts at such a check. See `docs/DUPLICATE_CHECK.md`.
+
- `run --self-repair` (and repair by default) now works outside a browser. `repair.session` in `deploy.yaml` says how to start the app for each probe, fault probe and resumed run (an argv, no shell, kept running while the app is up; `fault_commands`, `close_command`, `ready_s`). The run's configured backend must bind to a window owned by that command's process group, so a session never binds to the halted session or a copy already open; Flow ends the group afterwards, and closes the halted session before each probe. This works where the bound window is on Flow's machine (macOS, and the local RDP or Citrix client window); Windows over its agent and FreeRDP are refused for now. Without `repair.session`, a desktop run refuses repair with that reason, as before.
- Signed write receipts. A verifier verdict, with its scope (stores read, time window, strength) and a plain list of what it did not check, is signed as a COSE_Sign1 in the IETF SCITT shape and appended to a local transparency log. `openadapt-flow receipt verify FILE --issuer-key --log-key` checks one offline and prints the not-checked list every time; a receipt that claims more than its scope is neither issued nor accepted. `verify_write(..., write_receipt=True)`, `issue_write_receipt(result)` and `openadapt-flow shadow --receipts` issue them; `receipt anchor` puts the digest in Rekor. Extra: `openadapt-flow[receipts]`. Design: `docs/WRITE_RECEIPTS.md`.
diff --git a/packages/flow/docs/DUPLICATE_CHECK.md b/packages/flow/docs/DUPLICATE_CHECK.md
index 72ec7d83a3..5e6130b5ea 100644
--- a/packages/flow/docs/DUPLICATE_CHECK.md
+++ b/packages/flow/docs/DUPLICATE_CHECK.md
@@ -8,6 +8,45 @@ compiler mines that check from the recording: the dialog's empty result is an
exact check on the click that opens the dialog and again before Confirm
(`compiler/empty_result_guard.py`).
+The compiler knows an English vocabulary for that statement. A statement in
+other words ("Keine Treffer gefunden.") is mined from two or more
+demonstrations instead. The dialog must be named, and it must add exactly one
+line besides its title and button row. That line must read the same in every
+demonstration although every value typed before the dialog differs. Then
+induction checks it the same way. The line is not mined when:
+
+- it changes with the search;
+- it carries a typed value, a number other than 0, or an email address;
+- it is a question ("Wirklich speichern?");
+- it comes from one demonstration only.
+
+The PHI check on the line runs only when the optional privacy scrubber is
+installed. The compile notes of each recording, and of a bundle `admit`
+induces, say why a line was not mined.
+When the demonstrations disagree on a mined check, induction refuses
+(`ambiguous_guard`). It never copies the first recording's check, and it
+never drops the check quietly.
+
+A check mined this way was chosen because the demonstrations agree, not
+because of what its words say. So qualification needs a `business_rules` case
+that halts at it.
+
+Limits:
+
+- The dialog's title comes from the browser recorder (its accessible name),
+ so desktop recordings use the vocabulary only.
+- Agreement alone can't tell the statement from other fixed text that is the
+ dialog's only line, such as an empty table's single header or an
+ instruction. That needs a recording where the search found rows, and there
+ is no such contrast check yet. The business-rule run is what catches it,
+ and the qualification campaign reads linear bundles only, so an induced
+ bundle that carries such a check can't be qualified yet.
+- Only typed and selected values count as the search. A record chosen by
+ clicking a list row is not seen: the same row clicked in every
+ demonstration doesn't stop the line from being mined.
+- The confirmation pair itself is still found from English button labels
+ ("Confirm", "Yes").
+
OpenEMR lists a chart that shares only a first name with the new patient. On
the lifecycle soak's synthetic worklist, most of those stops were false: the
listed chart was clearly someone else, and a person entered the item by hand.
diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py
index 2abfb44aa3..a30512371d 100644
--- a/packages/flow/openadapt_flow/__main__.py
+++ b/packages/flow/openadapt_flow/__main__.py
@@ -2176,18 +2176,22 @@ def _cmd_induce(args: argparse.Namespace) -> int:
# bundle case CLI-side keeps induce usable on artifacts the operator
# already has, without touching the library API.
from openadapt_flow import sealed_set
+ from openadapt_flow.compiler.fixup_passes import read_compile_notes
from openadapt_flow.compiler.induction import induce_program, validate_held_out
from openadapt_flow.ir import Workflow
traces: list = []
+ notes: list = []
sealed_inputs = False
for d in args.recording:
path = Path(d)
sealed_inputs = sealed_inputs or sealed_set.is_sealed(path)
if (path / "workflow.json").is_file() or (path / "workflow.json.enc").is_file():
traces.append(Workflow.load(path))
+ notes.append(read_compile_notes(path))
else:
traces.append(path)
+ notes.append(None)
if sealed_inputs and (args.report or args.probe_on):
# Both write the induced program (or probe bundles) outside any
# sealed set; refuse rather than write plaintext.
@@ -2206,7 +2210,7 @@ def _cmd_induce(args: argparse.Namespace) -> int:
return probed
result = probed
else:
- result = induce_program(traces, **spoken)
+ result = induce_program(traces, compile_notes=notes, **spoken)
print(result.render())
if args.report:
report_path = Path(args.report)
diff --git a/packages/flow/openadapt_flow/admit_pipeline.py b/packages/flow/openadapt_flow/admit_pipeline.py
index c7d7ee9805..2895bd4556 100644
--- a/packages/flow/openadapt_flow/admit_pipeline.py
+++ b/packages/flow/openadapt_flow/admit_pipeline.py
@@ -291,6 +291,10 @@ def prepare_admission(
"""
from openadapt_flow.compiler import compile_recording
+ from openadapt_flow.compiler.fixup_passes import (
+ read_compile_notes,
+ write_compile_notes,
+ )
from openadapt_flow.compiler.induction import induce_program
from openadapt_flow.ir import Workflow
from openadapt_flow.qualification_proposal import (
@@ -366,10 +370,14 @@ def stop(status: PendingStatus, reason: str, **extra: Any) -> PendingAdmission:
bundle_dir = target / BUNDLE_DIR
induced = False
+ guard_notes: list[dict[str, str]] = []
if len(compiled) == 1:
workflow = Workflow.load(compiled[0])
else:
- result = induce_program([Workflow.load(p) for p in compiled])
+ result = induce_program(
+ [Workflow.load(p) for p in compiled],
+ compile_notes=[read_compile_notes(p) for p in compiled],
+ )
if not result.certified or result.workflow is None:
return stop(
"halted",
@@ -381,6 +389,7 @@ def stop(status: PendingStatus, reason: str, **extra: Any) -> PendingAdmission:
)
workflow = result.workflow
induced = True
+ guard_notes = list(result.guard_notes)
workflow = workflow.model_copy(update={"name": name})
from openadapt_flow import sealed_set
@@ -398,9 +407,12 @@ def stop(status: PendingStatus, reason: str, **extra: Any) -> PendingAdmission:
templates.setdefault(rel, sealed_set.read_bytes(item))
workflow._decrypted_templates = templates
with sealed_set.writing(bundle_dir, "bundle", passphrase):
+ write_compile_notes(bundle_dir, guard_notes)
workflow.save(bundle_dir)
else:
_copy_templates(compiled, bundle_dir)
+ # Induction's dialog-check decisions, as a compile's own notes.
+ write_compile_notes(bundle_dir, guard_notes)
workflow.save(bundle_dir)
workflow = Workflow.load(bundle_dir)
diff --git a/packages/flow/openadapt_flow/compiler/compile.py b/packages/flow/openadapt_flow/compiler/compile.py
index 45069a7aab..e941ffed72 100644
--- a/packages/flow/openadapt_flow/compiler/compile.py
+++ b/packages/flow/openadapt_flow/compiler/compile.py
@@ -38,6 +38,7 @@
from openadapt_flow.compiler.empty_result_guard import (
MinedGuard,
carries_value,
+ dialog_statement,
has_text_check,
is_empty_result_statement,
mine_empty_result_guard,
@@ -70,6 +71,7 @@
confirmation_opener_step_ids,
infer_step_risk,
is_meaningful_label,
+ opener_explanation,
presumed_commit_step_ids,
)
from openadapt_flow.runtime.identity import (
@@ -808,6 +810,20 @@ def _matches_label(text: str, labels: tuple[str, ...]) -> bool:
return False
+def _clicked_labels(steps: Sequence[Step]) -> tuple[str, ...]:
+ """The labels of every control the demonstration clicked."""
+
+ labels: list[str] = []
+ for step in steps:
+ if step.anchor is None:
+ continue
+ if step.anchor.ocr_text:
+ labels.append(step.anchor.ocr_text)
+ if step.anchor.structural is not None and step.anchor.structural.name:
+ labels.append(step.anchor.structural.name)
+ return tuple(labels)
+
+
def _text_carries_phi(text: str) -> bool:
"""Whether the optional Presidio pass flags this text as carrying PII/PHI.
@@ -3203,9 +3219,7 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]:
for opener_id, confirm_id in confirmation_opener_step_ids(steps):
opener = by_step_id[opener_id]
opener.risk = "reversible"
- opener.risk_explanation = (
- f"opens a confirmation dialog; the save is the next click ({confirm_id})"
- )
+ opener.risk_explanation = opener_explanation(confirm_id)
opener.risk_review_required = False
confirm = by_step_id[confirm_id]
confirm.risk_explanation = (
@@ -3216,17 +3230,43 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]:
# before the Confirm click. A pair with no such check is noted.
opener_before, opener_after, opener_event = frames_by_step_id[opener_id]
confirm_before, _, confirm_event = frames_by_step_id[confirm_id]
+ # A line in other words than the English vocabulary: kept as a
+ # candidate for induction, which checks it across demonstrations.
+ dialog_line: Optional[str] = None
+ dialog_line_note = ""
if opener_before is None or opener_after is None or confirm_before is None:
dialog_check = MinedGuard(note="the recording lacks the dialog's frames")
else:
- dialog_check = mine_empty_result_guard(
+ dialog_frames = (
cached_lines(int(opener_event["i"]), "before", opener_before),
cached_lines(int(opener_event["i"]), "after", opener_after),
cached_lines(int(confirm_event["i"]), "before", confirm_before),
+ )
+ dialog_check = mine_empty_result_guard(
+ *dialog_frames,
reject=lambda text: (
carries_value(text, exclude_texts) or _text_carries_phi(text)
),
)
+ section = confirm_event.get("section")
+ if dialog_check.guard is None and "x" in confirm_event:
+ dialog_line, dialog_line_note = dialog_statement(
+ *dialog_frames,
+ confirm_point=(int(confirm_event["x"]), int(confirm_event["y"])),
+ title=(
+ str(section.get("text") or "")
+ if isinstance(section, dict) and section.get("kind") == "dialog"
+ else None
+ ),
+ labels=_clicked_labels(steps),
+ # Not built from a fixed vocabulary: the fuzzy value screen
+ # too, which also catches a value OCR read without spaces.
+ reject=lambda text: (
+ carries_value(text, exclude_texts)
+ or _contains_excluded(text, exclude_texts)
+ or _text_carries_phi(text)
+ ),
+ )
if dialog_check.postcondition is not None and dialog_check.guard is not None:
if not has_text_check(opener.expect, dialog_check.postcondition.text or ""):
opener.expect = [*opener.expect, dialog_check.postcondition]
@@ -3248,14 +3288,17 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]:
confirm_id,
dialog_check.note,
)
- guard_notes.append(
- {
- "change": "no_empty_result_guard",
- "step": opener_id,
- "confirm_step": confirm_id,
- "warning": f"no check that the dialog found nothing: {dialog_check.note}",
- }
- )
+ missed: dict[str, Any] = {
+ "change": "no_empty_result_guard",
+ "step": opener_id,
+ "confirm_step": confirm_id,
+ "warning": f"no check that the dialog found nothing: {dialog_check.note}",
+ }
+ if dialog_line is not None:
+ missed["dialog_line"] = dialog_line
+ elif dialog_line_note:
+ missed["dialog_line_note"] = dialog_line_note
+ guard_notes.append(missed)
# Local edits the recording proves stayed local (select-all before
# typing, Enter on a typed dropdown option, an in-form tab): classified
diff --git a/packages/flow/openadapt_flow/compiler/empty_result_guard.py b/packages/flow/openadapt_flow/compiler/empty_result_guard.py
index b805f294a8..0dc3ce8ccd 100644
--- a/packages/flow/openadapt_flow/compiler/empty_result_guard.py
+++ b/packages/flow/openadapt_flow/compiler/empty_result_guard.py
@@ -36,6 +36,17 @@
OCR match): a ``TEXT_PRESENT`` postcondition of the opener, and a ``TEXT_PRESENT``
guard on the confirm step, checked on the screen right before the Confirm
click (an early empty state that a slower result replaces is caught there).
+
+The vocabulary is English. A statement in any other words is found by
+:func:`dialog_statement` instead, which knows no words at all: it takes the
+one line the dialog added (not its title, not its button row), when that line
+is confidently read, shown once, not a question, and carries no run value,
+number other than 0, email address, or PHI the optional scrubber finds. One demonstration cannot tell such a line from
+record data, so a compile keeps it only as a candidate in its
+``no_empty_result_guard`` note. Induction mines it when two or more
+demonstrations whose search inputs differ all show the same line
+(:mod:`openadapt_flow.compiler.induction`): record data changes with the
+search, the dialog's own words do not.
"""
from __future__ import annotations
@@ -173,14 +184,21 @@ def mine_empty_result_guard(
note="the dialog's empty search result carries a run value or PHI",
statements=shown,
)
- return MinedGuard(
- postcondition=Postcondition(
+ postcondition, guard = exact_checks(text)
+ return MinedGuard(postcondition=postcondition, guard=guard, statements=shown)
+
+
+def exact_checks(text: str) -> tuple[Postcondition, Guard]:
+ """The opener's postcondition and the confirm step's guard for ``text``."""
+
+ return (
+ Postcondition(
kind=PostconditionKind.TEXT_PRESENT,
text=text,
timeout_s=GUARD_TIMEOUT_S,
exact=True,
),
- guard=Guard(
+ Guard(
predicate=Predicate(
kind=PredicateKind.TEXT_PRESENT,
text=text,
@@ -189,10 +207,110 @@ def mine_empty_result_guard(
),
on_unmet="halt",
),
- statements=shown,
)
+def looks_like_data(text: str) -> bool:
+ """Whether ``text`` carries a number other than 0, or an email address.
+
+ Counts, dates, ids and addresses change from record to record; a dialog's
+ own statement of an empty result needs none of them ("0 Treffer" is fine).
+ """
+
+ if "@" in text:
+ return True
+ return any(run != "0" for run in re.findall(r"\d+", text))
+
+
+def _overlaps_vertically(a: OcrLine, b: OcrLine) -> bool:
+ _ax, ay, _aw, ah = a.region
+ _bx, by, _bw, bh = b.region
+ return ay < by + bh and by < ay + ah
+
+
+def _contains(line: OcrLine, point: tuple[int, int]) -> bool:
+ x, y, w, h = line.region
+ return x <= point[0] <= x + w and y <= point[1] <= y + h
+
+
+def _exact_count(lines: Sequence[OcrLine], key: str) -> int:
+ return sum(
+ 1
+ for readings in wrapped_readings(lines, min_confidence=MIN_CONFIDENCE)
+ if any(text_key(reading) == key for reading in readings)
+ )
+
+
+def dialog_statement(
+ opener_before: Sequence[OcrLine],
+ opener_after: Sequence[OcrLine],
+ confirm_before: Sequence[OcrLine],
+ *,
+ confirm_point: tuple[int, int],
+ title: Optional[str],
+ labels: Sequence[str] = (),
+ reject: Callable[[str], bool] = lambda _text: False,
+) -> tuple[Optional[str], str]:
+ """The one line a dialog added, in any language, or why there is none.
+
+ The line is what the dialog shows besides its title and its button row:
+ every line on the Confirm screen that was not on the screen before the
+ opener click, other than ``title`` (the dialog's accessible name, recorded
+ by the browser), the lines level with the Confirm button (``Cancel``
+ beside it), and ``labels`` (the names of controls the demonstration
+ clicked). It must be exactly one line, or one line wrapped over up to
+ three, every part read at confidence 0.8 or more; shown once on the
+ Confirm screen and at most once after the opener; free of numbers other
+ than 0 and email addresses (:func:`looks_like_data`); not a question
+ ("Are you sure?"); and ``reject`` (run values, and PHI when the optional
+ privacy scrubber is installed) must not refuse it.
+
+ Returns:
+ ``(line, "")``, or ``(None, reason)``. A line returned here is only a
+ candidate: nothing on one screen tells a dialog's statement from a
+ record it lists. It becomes a check only with the evidence induction
+ adds (the same line across demonstrations with different searches).
+ """
+
+ if not title or not text_key(title):
+ return None, "the recording does not name the dialog, so its title is unknown"
+ rows = [line for line in confirm_before if _contains(line, confirm_point)]
+ if not rows:
+ return None, "the Confirm button's label was not read"
+ before = {
+ text_key(reading)
+ for readings in wrapped_readings(opener_before)
+ for reading in readings
+ }
+ skip = {text_key(title), *(text_key(label) for label in labels)}
+ added = [
+ line
+ for line in confirm_before
+ if text_key(line.text)
+ and text_key(line.text) not in before
+ and text_key(line.text) not in skip
+ and not any(_overlaps_vertically(line, row) for row in rows)
+ ]
+ if not added:
+ return None, "the dialog shows no line besides its title and buttons"
+ if any(line.confidence < MIN_CONFIDENCE for line in added):
+ return None, "a line the dialog shows is not confidently read"
+ top = min(added, key=lambda line: (line.region[1], line.region[0]))
+ readings = wrapped_readings([top, *(line for line in added if line is not top)])
+ if len(readings[0]) != len(added):
+ return None, f"the dialog shows {len(added)} lines, not one statement"
+ text = readings[0][-1]
+ key = text_key(text)
+ if _exact_count(confirm_before, key) != 1 or _exact_count(opener_after, key) > 1:
+ return None, "the dialog's line is shown more than once"
+ if looks_like_data(text) or reject(text):
+ return None, "the dialog's line carries a run value, PHI or a number"
+ if text.rstrip().endswith(("?", "\uff1f", "\u061f")):
+ # "Are you sure?" stays on screen whatever the search found.
+ return None, "the dialog's line is a question, not a search result"
+ return text, ""
+
+
def carries_value(text: str, values: Sequence[str]) -> bool:
"""Whether ``text`` contains one of ``values`` as whole words.
@@ -243,9 +361,12 @@ def is_empty_result_check(pc: Postcondition) -> bool:
"GUARD_TIMEOUT_S",
"MinedGuard",
"carries_value",
+ "dialog_statement",
"empty_result_statements",
+ "exact_checks",
"has_text_check",
"is_empty_result_check",
"is_empty_result_statement",
+ "looks_like_data",
"mine_empty_result_guard",
]
diff --git a/packages/flow/openadapt_flow/compiler/fixup_passes.py b/packages/flow/openadapt_flow/compiler/fixup_passes.py
index 9b6c7b0419..8e165ea58c 100644
--- a/packages/flow/openadapt_flow/compiler/fixup_passes.py
+++ b/packages/flow/openadapt_flow/compiler/fixup_passes.py
@@ -364,9 +364,24 @@ def write_compile_notes(bundle_dir: Path, notes: list[dict[str, Any]]) -> None:
)
+def read_compile_notes(bundle_dir: Path) -> list[dict[str, Any]]:
+ """The entries of a bundle's ``compile_notes.json`` (none when absent)."""
+
+ import json
+
+ from openadapt_flow import sealed_set
+
+ path = Path(bundle_dir) / COMPILE_NOTES_FILE
+ if not sealed_set.is_file(path):
+ return []
+ changes = json.loads(sealed_set.read_text(path)).get("changes")
+ return [note for note in changes or [] if isinstance(note, dict)]
+
+
__all__ = [
"COMPILE_NOTES_FILE",
"apply_fixup_passes",
+ "read_compile_notes",
"is_select_all",
"region_unchanged",
"write_compile_notes",
diff --git a/packages/flow/openadapt_flow/compiler/induction.py b/packages/flow/openadapt_flow/compiler/induction.py
index d7fdff8593..866f111b6a 100644
--- a/packages/flow/openadapt_flow/compiler/induction.py
+++ b/packages/flow/openadapt_flow/compiler/induction.py
@@ -93,6 +93,7 @@
from __future__ import annotations
+import functools
import tempfile
import warnings
from difflib import SequenceMatcher
@@ -213,7 +214,7 @@ class Uncertainty(BaseModel):
"'ambiguous_branch' | 'ambiguous_loop' | 'unconfirmed_branch' | "
"'unconfirmed_optional' | 'underdetermined_order' | "
"'ambiguous_selection' | 'alignment_failure' | "
- "'unconfirmed_loop_branch'"
+ "'unconfirmed_loop_branch' | 'ambiguous_guard'"
)
)
location: str = Field(description="Column / position the ambiguity is at")
@@ -414,6 +415,14 @@ class InductionResult(BaseModel):
default_factory=list,
description="Bound and per-row checks of every emitted loop.",
)
+ guard_notes: list[dict[str, str]] = Field(
+ default_factory=list,
+ description=(
+ "Dialog empty-result checks (compiler.empty_result_guard): each "
+ "one the demonstrations agreed on (empty_result_guard), and each "
+ "one not added and why (no_empty_result_guard)."
+ ),
+ )
@property
def underdetermined(self) -> bool:
@@ -519,24 +528,31 @@ def render(self) -> str:
# ===========================================================================
-def _as_workflow(trace: TraceInput) -> Workflow:
+def _as_workflow(
+ trace: TraceInput, notes: Optional[list[dict[str, Any]]] = None
+) -> Workflow:
"""Normalize a trace to a linear ``Workflow``. A recording DIRECTORY is
compiled through the single-trace bootstrap ``compile_recording`` (RFC §3
- [1]); a ``Workflow`` is used as-is."""
+ [1]); a ``Workflow`` is used as-is. ``notes``, when given, receives the
+ bootstrap compile's ``compile_notes.json`` entries."""
if isinstance(trace, Workflow):
return trace
from openadapt_flow.compiler.compile import compile_recording
+ from openadapt_flow.compiler.fixup_passes import read_compile_notes
recording = Path(trace)
with tempfile.TemporaryDirectory(prefix="induce-boot-") as tmp:
# Identity bands stay readable until induction has named every row
# column; induce_program hashes them before returning.
- return compile_recording(
+ workflow = compile_recording(
recording,
Path(tmp) / "bundle",
name=recording.name,
phi_free_identity=False,
)
+ if notes is not None:
+ notes.extend(read_compile_notes(Path(tmp) / "bundle"))
+ return workflow
def phi_free_program_identity(workflow: Workflow) -> None:
@@ -672,10 +688,21 @@ def _trace_sigs(steps: list[Step]) -> list[tuple[str, str]]:
then a click on the result) selects the record that value names. Its
label differs row to row, so its signature is the value-free
``select:typed`` instead of the label.
+
+ The Confirm click of a confirmation dialog
+ (:func:`openadapt_flow.risk.compiled_confirmation_pairs`) is marked
+ ``confirm:``: the opener and its Confirm are two buttons in a row, not two
+ rows of a loop.
"""
+ from openadapt_flow.risk import compiled_confirmation_pairs
+
+ confirm_ids = {confirm for _opener, confirm in compiled_confirmation_pairs(steps)}
sigs: list[tuple[str, str]] = []
for i, step in enumerate(steps):
sig = _sig(step)
+ if step.id in confirm_ids:
+ sigs.append((sig[0], f"confirm:{sig[1]}"))
+ continue
if step.action in (ActionKind.CLICK, ActionKind.DOUBLE_CLICK) and not sig[
1
].startswith("select:"):
@@ -1279,6 +1306,7 @@ def induce_program(
interpreter: Optional[Any] = None,
accepted_narration: Optional[dict[str, str]] = None,
accepted_loop_guards: Optional[dict[str, str]] = None,
+ compile_notes: Optional[Sequence[Optional[Sequence[dict[str, Any]]]]] = None,
) -> InductionResult:
"""Induce a parameterized :class:`ProgramGraph` from multiple demonstrations
of the same task -- the RFC §3 induction loop (step [4] + [5]).
@@ -1320,12 +1348,22 @@ def induce_program(
-- a write a loop body makes for some rows only, run when that
screen text shows. The report proposes the text; a person
accepts it. Without acceptance such a loop is refused.
+ compile_notes: One list of ``compile_notes.json`` entries (or None)
+ per trace, for traces given as compiled ``Workflow``s. A recording
+ directory's come from its bootstrap compile. They carry each
+ dialog's candidate empty-result line (``dialog_line``), which is
+ mined only when every demonstration shows the same one.
Returns:
An :class:`InductionResult` with the induced program (or a refusal) and
the full audit trail.
"""
- workflows = [_as_workflow(t) for t in traces]
+ trace_notes: list[list[dict[str, Any]]] = [[] for _ in traces]
+ workflows = [_as_workflow(t, trace_notes[i]) for i, t in enumerate(traces)]
+ for i, given in enumerate(compile_notes or ()):
+ # A recording directory's notes come from its own bootstrap compile.
+ if i < len(traces) and isinstance(traces[i], Workflow) and given:
+ trace_notes[i].extend(given)
n = len(workflows)
result = InductionResult(n_traces=n)
if n == 0:
@@ -1364,6 +1402,7 @@ def induce_program(
divergences: dict[int, tuple[list[_Column], int]] = {}
# Loop identity columns no demonstration shows (each run's rows name them).
identity_only: set[str] = set()
+ singles: dict[int, _Node] = {}
for idx, col in enumerate(columns):
present = sorted(col.tokens)
@@ -1410,6 +1449,8 @@ def induce_program(
)
nodes.append(node)
decisions.append(dec)
+ if col.kind == "single":
+ singles[idx] = node
continue
# --- single step present in SOME traces: branch or optional -------
@@ -1421,6 +1462,10 @@ def induce_program(
optional_nodes[idx] = node
decisions.append(dec)
+ _reconcile_dialog_guards(
+ columns, singles, workflows, trace_notes, decisions, result
+ )
+
guard_sources: dict[str, GuardSource] = {}
if narration and any(narration):
_apply_narration(
@@ -1473,6 +1518,232 @@ def induce_program(
return result
+def _exact_guard_text(step: Step) -> Optional[str]:
+ """The exact text a step's guard checks (a mined dialog check), if any."""
+
+ guard = step.guard
+ if (
+ guard is not None
+ and guard.predicate.kind is PredicateKind.TEXT_PRESENT
+ and guard.predicate.exact
+ and guard.predicate.text
+ ):
+ return guard.predicate.text
+ return None
+
+
+def _dialog_line(notes: Sequence[dict[str, Any]], opener_id: str) -> Optional[str]:
+ """The candidate line a trace's compile kept for the dialog ``opener_id`` opens."""
+
+ for note in notes:
+ if (
+ note.get("change") == "no_empty_result_guard"
+ and note.get("step") == opener_id
+ ):
+ line = note.get("dialog_line")
+ return line if isinstance(line, str) and line else None
+ return None
+
+
+def _col_steps(col: _Column) -> list[Step]:
+ """Every step of every trace at one aligned column (loop rows included)."""
+
+ return [step for tok in col.tokens.values() for step in _tok_steps(tok)]
+
+
+def _guard_note(
+ result: InductionResult, opener_sid: str, confirm_sid: str, change: str, why: str
+) -> None:
+ entry = {"change": change, "step": opener_sid, "confirm_step": confirm_sid}
+ entry["evidence" if change == "empty_result_guard" else "warning"] = why
+ result.guard_notes.append(entry)
+
+
+def _reconcile_dialog_guards(
+ columns, singles, workflows, trace_notes, decisions, result
+):
+ """Decide each confirmation dialog's empty-result check from every trace.
+
+ A step is copied from the first trace that has it, and with it the check
+ that trace's compile mined. That check is kept only when every trace
+ mined the same one. When they differ, induction refuses
+ (``ambiguous_guard``): dropping the check would let a run click through a
+ dialog that lists a match, and keeping it would pick one trace's dialog.
+
+ When no trace mined one, the candidate line each trace's compile kept
+ (``dialog_line``, in any language) becomes the check when every trace has
+ the same line, there are two or more traces, and every value typed
+ before the opener differs between them. Record data changes with the
+ search; the dialog's own statement does not. A value chosen by a click
+ (a list row) is not seen here: the same row clicked in every
+ demonstration does not stop the line from being mined.
+ """
+
+ from openadapt_flow.compiler.empty_result_guard import exact_checks
+ from openadapt_flow.risk import compiled_confirmation_pairs
+ from openadapt_flow.vision.ocr import text_key
+
+ n = len(workflows)
+ pairs = [set(compiled_confirmation_pairs(wf.steps)) for wf in workflows]
+ kinds = {dec.index: dec.kind for dec in decisions}
+ handled: set[int] = set()
+ for idx in sorted(singles):
+ if idx - 1 not in singles:
+ continue
+ openers = [columns[idx - 1].tokens[t].step for t in range(n)]
+ confirms = [columns[idx].tokens[t].step for t in range(n)]
+ paired = [(openers[t].id, confirms[t].id) in pairs[t] for t in range(n)]
+ if not any(paired):
+ continue
+ handled.update((idx - 1, idx))
+ opener_sid, confirm_sid = f"s{idx - 1}", f"s{idx}"
+ opener = singles[idx - 1].states[opener_sid].step
+ confirm = singles[idx].states[confirm_sid].step
+ texts = [_exact_guard_text(step) for step in confirms]
+ keys = {text_key(text) if text else "" for text in texts}
+ note = functools.partial(_guard_note, result, opener.id, confirm.id)
+ if any(texts):
+ if all(paired) and len(keys) == 1:
+ note(
+ "empty_result_guard",
+ f"every one of the {n} demonstrations showed the same "
+ "empty-result statement",
+ )
+ continue
+ mined = [t for t in range(n) if texts[t]]
+ why = (
+ f"the demonstrations disagree on the dialog's empty-result "
+ f"check (recordings {mined} mined one, "
+ f"{len({k for k in keys if k})} different; the others none)"
+ )
+ note("no_empty_result_guard", f"induction refused: {why}")
+ _add_uncertainty(
+ result,
+ kind="ambiguous_guard",
+ location=confirm_sid,
+ detail=why,
+ consequential=True,
+ )
+ continue
+ if not all(paired):
+ note(
+ "no_empty_result_guard",
+ "no check that the dialog found nothing: the click opens a "
+ "confirmation dialog in only some demonstrations",
+ )
+ continue
+ lines = [_dialog_line(trace_notes[t], openers[t].id) for t in range(n)]
+ line_keys = {text_key(line) for line in lines if line}
+ # Every value typed before the opener: the search is one of them,
+ # and any of them may feed the dialog.
+ typed = [
+ j
+ for j in range(idx - 1)
+ if any(
+ step.action in (ActionKind.TYPE, ActionKind.SELECT_OPTION)
+ for step in _col_steps(columns[j])
+ )
+ ]
+ why = ""
+ if n < 2:
+ why = "one demonstration cannot tell the dialog's line from record data"
+ elif not all(lines):
+ why = "not every demonstration's dialog showed one candidate line"
+ elif len(line_keys) != 1:
+ why = "the dialog's line differs between demonstrations (record data)"
+ elif not typed or any(kinds.get(j) != "param" for j in typed):
+ why = (
+ "a value typed before the dialog is the same in every "
+ "demonstration, so a line that stays the same proves nothing"
+ )
+ elif confirm.guard is not None:
+ why = "the Confirm step already has a guard"
+ if why:
+ note(
+ "no_empty_result_guard",
+ f"no check that the dialog found nothing: {why}",
+ )
+ continue
+ text = lines[0]
+ assert text is not None
+ postcondition, guard = exact_checks(text)
+ opener.expect = [*opener.expect, postcondition]
+ confirm.guard = guard
+ note(
+ "empty_result_guard",
+ f"the same dialog line in all {n} demonstrations, while every "
+ "value typed before the dialog differed between them; checked "
+ "after the opener and before the Confirm click",
+ )
+ _refuse_unreconciled_guards(columns, handled, result, pairs)
+
+
+def _refuse_unreconciled_guards(columns, handled, result, pairs=None) -> None:
+ """Refuse a mined dialog check the traces disagree on off the straight path.
+
+ An optional, divergent or loop column copies one trace's step, and with
+ it that trace's check. It is kept only when every step at that place, in
+ every trace and every loop row, has the same check (or none has one).
+ A Confirm click there with no check is noted (``no_empty_result_guard``).
+ """
+
+ confirms = {
+ (t, confirm): opener
+ for t, trace_pairs in enumerate(pairs or [])
+ for opener, confirm in trace_pairs
+ }
+
+ from openadapt_flow.vision.ocr import text_key
+
+ def checks(steps: Sequence[Step]) -> tuple[str, ...]:
+ # A Confirm click's guard, and an opener's exact postcondition.
+ return tuple(
+ text_key(_exact_guard_text(step) or "")
+ + "|"
+ + ",".join(
+ sorted(text_key(pc.text or "") for pc in step.expect if pc.exact)
+ )
+ for step in steps
+ )
+
+ for idx, col in enumerate(columns):
+ if idx in handled:
+ continue
+ shapes: set[tuple[str, ...]] = set()
+ for tok in col.tokens.values():
+ rows = [[tok.step]] if tok.kind == "single" else tok.iterations
+ shapes.update(checks(row) for row in rows)
+ if all(check == "|" for shape in shapes for check in shape):
+ # One note per column, naming the step the program copied (the
+ # first trace's, its first row's).
+ t, tok = min(col.tokens.items())
+ first = [[tok.step]] if tok.kind == "single" else tok.iterations[:1]
+ for step in (step for row in first for step in row):
+ if (t, step.id) in confirms:
+ _guard_note(
+ result,
+ confirms[(t, step.id)],
+ step.id,
+ "no_empty_result_guard",
+ "no check that the dialog found nothing: the dialog "
+ "is not on every demonstration's straight path",
+ )
+ continue
+ if len(shapes) == 1:
+ continue
+ _add_uncertainty(
+ result,
+ kind="ambiguous_guard",
+ location=f"s{idx}",
+ detail=(
+ "the demonstrations disagree on a confirmation dialog's "
+ "empty-result check in a step not every trace or row repeats "
+ "the same way"
+ ),
+ consequential=True,
+ )
+
+
def _emit_loop(
idx,
col,
diff --git a/packages/flow/openadapt_flow/qualify/campaign.py b/packages/flow/openadapt_flow/qualify/campaign.py
index df6944df1e..00f2866bdf 100644
--- a/packages/flow/openadapt_flow/qualify/campaign.py
+++ b/packages/flow/openadapt_flow/qualify/campaign.py
@@ -1792,6 +1792,47 @@ def authorize(
return result
+def _all_steps(workflow: Workflow) -> list[Any]:
+ """The linear steps and every program-graph step."""
+
+ graphs = [
+ *([workflow.program] if workflow.program is not None else []),
+ *workflow.subflows.values(),
+ ]
+ return [
+ *workflow.steps,
+ *(
+ state.step
+ for graph in graphs
+ for state in graph.states.values()
+ if state.step is not None
+ ),
+ ]
+
+
+def _dialog_guard_texts(workflow: Workflow) -> dict[str, str]:
+ """``{confirm step id: text key}`` of each exact halt guard on a Confirm
+ click of a confirmation dialog (the checks ``empty_result_guard`` mines)."""
+
+ from openadapt_flow.ir import PredicateKind
+ from openadapt_flow.risk import is_confirm_click
+ from openadapt_flow.vision.ocr import text_key
+
+ out: dict[str, str] = {}
+ for step in _all_steps(workflow):
+ guard = step.guard
+ if (
+ guard is not None
+ and guard.on_unmet == "halt"
+ and guard.predicate.kind is PredicateKind.TEXT_PRESENT
+ and guard.predicate.exact
+ and guard.predicate.text
+ and is_confirm_click(step)
+ ):
+ out[step.id] = text_key(guard.predicate.text)
+ return out
+
+
def rule_halt_steps(
workflow: Workflow, rule: BusinessRuleCase, save_id: str
) -> set[str]:
@@ -1799,25 +1840,27 @@ def rule_halt_steps(
from openadapt_flow.compiler.empty_result_guard import is_empty_result_statement
from openadapt_flow.ir import PostconditionKind
+ from openadapt_flow.vision.ocr import text_key
ids = [step.id for step in workflow.steps]
if rule.halt_at == "empty_result_guard":
- # The compiler's exact checks, and an empty-result text check a bundle
- # compiled before them carries (added by hand after compiling).
+ # The compiler's exact checks on a confirmation dialog (in any
+ # language), and an empty-result text check a bundle compiled before
+ # them carries (added by hand after compiling).
+ guards = _dialog_guard_texts(workflow)
return {
step.id
- for step in workflow.steps
- if any(
+ for step in _all_steps(workflow)
+ if step.id in guards
+ or any(
pc.kind is PostconditionKind.TEXT_PRESENT
- and is_empty_result_statement(pc.text or "")
+ and bool(pc.text)
+ and (
+ is_empty_result_statement(pc.text or "")
+ or (pc.exact and text_key(pc.text or "") in guards.values())
+ )
for pc in step.expect
)
- or (
- step.guard is not None
- and step.guard.on_unmet == "halt"
- and step.guard.predicate.exact
- and is_empty_result_statement(step.guard.predicate.text or "")
- )
}
if rule.halt_at == "before_save":
# The save step itself only before its action (its guard or identity
@@ -1831,7 +1874,28 @@ def business_rule_problems(
) -> list[str]:
"""What keeps the business-rule cases from running, found before any run."""
+ from openadapt_flow.compiler.empty_result_guard import is_empty_result_statement
+
problems: list[str] = []
+ # A dialog check in other words than the English vocabulary was mined
+ # because every demonstration showed it, not because of what it says. A
+ # run on a record the search finds must halt at it before it counts.
+ dialog_guards = _dialog_guard_texts(workflow)
+ unworded = sorted(
+ step.id
+ for step in _all_steps(workflow)
+ if step.id in dialog_guards
+ and step.guard is not None
+ and not is_empty_result_statement(step.guard.predicate.text or "")
+ )
+ if unworded and not any(
+ rule.halt_at == "empty_result_guard" for rule in config.business_rules
+ ):
+ problems.append(
+ f"steps {unworded} check a dialog line the demonstrations agreed "
+ "on; add a business_rules case (halt_at: empty_result_guard) whose "
+ "search finds a record, so a run proves the check halts"
+ )
for rule in config.business_rules:
unknown = sorted(set(rule.params) - set(params))
if unknown:
diff --git a/packages/flow/openadapt_flow/risk.py b/packages/flow/openadapt_flow/risk.py
index 687d5c8b0c..55e574311c 100644
--- a/packages/flow/openadapt_flow/risk.py
+++ b/packages/flow/openadapt_flow/risk.py
@@ -594,6 +594,47 @@ def confirmation_opener_step_ids(steps: Iterable[Step]) -> tuple[tuple[str, str]
return tuple(pairs)
+def is_confirm_click(step: Step) -> bool:
+ """Whether ``step`` is an irreversible click labelled "Confirm" or "Yes"."""
+
+ return (
+ step.action in _POINTER_ACTIONS
+ and step.risk == "irreversible"
+ and _CONFIRMATION_LABEL.search(step_text(step)) is not None
+ )
+
+
+def opener_explanation(confirm_id: str) -> str:
+ """The risk explanation the compiler gives a click that opens a dialog."""
+
+ return f"opens a confirmation dialog; the save is the next click ({confirm_id})"
+
+
+def compiled_confirmation_pairs(
+ steps: Iterable[Step],
+) -> tuple[tuple[str, str], ...]:
+ """``(opener_id, confirm_id)`` pairs, before or after compiling.
+
+ :func:`confirmation_opener_step_ids` reads the per-step risk inference,
+ which the compiler then changes: the opener becomes reversible, with
+ :func:`opener_explanation` as its reason. A compiled program's pairs are
+ the steps that carry that reason, followed by the irreversible click it
+ names.
+ """
+
+ ordered = list(steps)
+ inferred = set(confirmation_opener_step_ids(ordered))
+ return tuple(
+ (opener.id, confirm.id)
+ for opener, confirm in zip(ordered, ordered[1:])
+ if (opener.id, confirm.id) in inferred
+ or (
+ confirm.risk == "irreversible"
+ and opener.risk_explanation == opener_explanation(confirm.id)
+ )
+ )
+
+
def step_text(step: Step) -> str:
"""The text a click step's risk is inferred from: its intent plus its
target's retained OCR and structural-accessibility labels (the intent
diff --git a/packages/flow/tests/fixtures/search_dialog/README.md b/packages/flow/tests/fixtures/search_dialog/README.md
new file mode 100644
index 0000000000..f0cc12c57a
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/README.md
@@ -0,0 +1,22 @@
+Recordings of a small search-before-save page (`app.html`), made with the real
+recorder in headless Chromium by `record.py` (2026-11-03). Each one types a
+name, clicks "Create record", which opens a dialog named "Dublettensuche", and
+clicks the dialog's "Confirm". What the dialog says is set by the `s` query
+parameter:
+
+- `de_marsh`, `de_weber`, `de_marsh_again`: "Keine Treffer gefunden." (the
+ last one repeats the first one's name);
+- `archive_marsh`, `archive_weber`: "Keine Treffer im Archiv Nord" or "West",
+ chosen by the typed name (a line that changes with the data);
+- `names_marsh`: "Keine Treffer für Elena Marsh" (the typed value);
+- `en_weber`: "No results found" (the English vocabulary);
+- `question_marsh`: "Wirklich speichern?" (a question, shown whatever the
+ search found);
+- `comment_marsh_first`, `comment_marsh_second`: the archive line, the same
+ name both times, and a different comment typed after it into a field
+ labelled "Note" (`?c=1` shows that field).
+
+The buttons are in English because the confirmation pair is found from the
+Confirm label. The page URL is replaced with `http://records.test/new` in the
+events. The names are made up. Used by
+`tests/test_empty_result_guard_evidence.py`.
diff --git a/packages/flow/tests/fixtures/search_dialog/app.html b/packages/flow/tests/fixtures/search_dialog/app.html
new file mode 100644
index 0000000000..9d98a310d7
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/app.html
@@ -0,0 +1,52 @@
+
+
+
+
+Records
+
+
+
+
New record
+
+
+
+
Record saved
+
+
Dublettensuche
+
+
+
+
+
+
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/events.jsonl b/packages/flow/tests/fixtures/search_dialog/archive_marsh/events.jsonl
new file mode 100644
index 0000000000..d3f4e4a031
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/archive_marsh/events.jsonl
@@ -0,0 +1,4 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 0.937}
+{"i": 1, "kind": "type", "text": "Elena Marsh", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.723}
+{"i": 2, "kind": "click", "x": 113, "y": 199, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.391}
+{"i": 3, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 3.252}
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0000_after.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0000_before.png
new file mode 100644
index 0000000000..28a1f29e73
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0001_after.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0001_before.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0002_after.png
new file mode 100644
index 0000000000..618e073db2
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0002_before.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0003_after.png
new file mode 100644
index 0000000000..dd60851a8e
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0003_before.png
new file mode 100644
index 0000000000..618e073db2
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_marsh/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_marsh/meta.json b/packages/flow/tests/fixtures/search_dialog/archive_marsh/meta.json
new file mode 100644
index 0000000000..b7ebc94457
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/archive_marsh/meta.json
@@ -0,0 +1,14 @@
+{
+ "id": "867a5358c8604949b5da8d379134d200",
+ "created_at": "2026-11-03T19:49:57.780757+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Elena Marsh"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/events.jsonl b/packages/flow/tests/fixtures/search_dialog/archive_weber/events.jsonl
new file mode 100644
index 0000000000..d3a7945f0d
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/archive_weber/events.jsonl
@@ -0,0 +1,4 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.037}
+{"i": 1, "kind": "type", "text": "Jonas Weber", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.83}
+{"i": 2, "kind": "click", "x": 113, "y": 199, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.376}
+{"i": 3, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 3.02}
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0000_after.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0000_before.png
new file mode 100644
index 0000000000..28a1f29e73
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0001_after.png
new file mode 100644
index 0000000000..7650128874
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0001_before.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0002_after.png
new file mode 100644
index 0000000000..525fdc5621
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0002_before.png
new file mode 100644
index 0000000000..7650128874
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0003_after.png
new file mode 100644
index 0000000000..41de7e136c
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0003_before.png
new file mode 100644
index 0000000000..525fdc5621
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/archive_weber/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/archive_weber/meta.json b/packages/flow/tests/fixtures/search_dialog/archive_weber/meta.json
new file mode 100644
index 0000000000..403132f9e0
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/archive_weber/meta.json
@@ -0,0 +1,14 @@
+{
+ "id": "cd77332502ff4b019b6e1d54b65bfa98",
+ "created_at": "2026-11-03T19:50:17.256632+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Jonas Weber"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/events.jsonl b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/events.jsonl
new file mode 100644
index 0000000000..1917e281b4
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/events.jsonl
@@ -0,0 +1,6 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.601}
+{"i": 1, "kind": "type", "text": "Elena Marsh", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.549}
+{"i": 2, "kind": "click", "x": 286, "y": 173, "structural": {"selector": "#comment", "role": "textbox", "name": "Note"}, "structured_identity": "Note", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 3.052}
+{"i": 3, "kind": "type", "text": "Erster Anruf", "field_label": "Note", "param": "comment", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 3.676}
+{"i": 4, "kind": "click", "x": 113, "y": 250, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 2, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": false, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 4.207}
+{"i": 5, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 2, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": false, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 5.005}
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0000_after.png
new file mode 100644
index 0000000000..5517f24445
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0000_before.png
new file mode 100644
index 0000000000..91cf8ca40e
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0001_after.png
new file mode 100644
index 0000000000..fa58cf1a1f
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0001_before.png
new file mode 100644
index 0000000000..5517f24445
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0002_after.png
new file mode 100644
index 0000000000..275f68a6de
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0002_before.png
new file mode 100644
index 0000000000..fa58cf1a1f
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0003_after.png
new file mode 100644
index 0000000000..d44f1324cf
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0003_before.png
new file mode 100644
index 0000000000..275f68a6de
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0004_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0004_after.png
new file mode 100644
index 0000000000..28ab0f28d7
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0004_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0004_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0004_before.png
new file mode 100644
index 0000000000..d44f1324cf
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0004_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0005_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0005_after.png
new file mode 100644
index 0000000000..405f68980a
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0005_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0005_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0005_before.png
new file mode 100644
index 0000000000..28ab0f28d7
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/frames/0005_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/meta.json b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/meta.json
new file mode 100644
index 0000000000..36b804b022
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/comment_marsh_first/meta.json
@@ -0,0 +1,15 @@
+{
+ "id": "56cb61c6e31b4d37928269bebfa96ddf",
+ "created_at": "2026-11-04T00:06:45.789431+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Elena Marsh",
+ "comment": "Erster Anruf"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/events.jsonl b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/events.jsonl
new file mode 100644
index 0000000000..eadc65748c
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/events.jsonl
@@ -0,0 +1,6 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 0.901}
+{"i": 1, "kind": "type", "text": "Elena Marsh", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.572}
+{"i": 2, "kind": "click", "x": 286, "y": 173, "structural": {"selector": "#comment", "role": "textbox", "name": "Note"}, "structured_identity": "Note", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.577}
+{"i": 3, "kind": "type", "text": "Rueckruf morgen", "field_label": "Note", "param": "comment", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 3.211}
+{"i": 4, "kind": "click", "x": 113, "y": 250, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 2, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": false, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 3.95}
+{"i": 5, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 2, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": false, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 4.784}
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0000_after.png
new file mode 100644
index 0000000000..5517f24445
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0000_before.png
new file mode 100644
index 0000000000..91cf8ca40e
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0001_after.png
new file mode 100644
index 0000000000..fa58cf1a1f
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0001_before.png
new file mode 100644
index 0000000000..5517f24445
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0002_after.png
new file mode 100644
index 0000000000..275f68a6de
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0002_before.png
new file mode 100644
index 0000000000..fa58cf1a1f
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0003_after.png
new file mode 100644
index 0000000000..a3f3395d6c
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0003_before.png
new file mode 100644
index 0000000000..275f68a6de
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0004_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0004_after.png
new file mode 100644
index 0000000000..d9c43c8b1d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0004_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0004_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0004_before.png
new file mode 100644
index 0000000000..a3f3395d6c
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0004_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0005_after.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0005_after.png
new file mode 100644
index 0000000000..1f89973d64
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0005_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0005_before.png b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0005_before.png
new file mode 100644
index 0000000000..d9c43c8b1d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/frames/0005_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/meta.json b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/meta.json
new file mode 100644
index 0000000000..807b5c4931
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/comment_marsh_second/meta.json
@@ -0,0 +1,15 @@
+{
+ "id": "e021f56d368b4265bee161b4c8076047",
+ "created_at": "2026-11-04T00:07:07.796511+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Elena Marsh",
+ "comment": "Rueckruf morgen"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/events.jsonl b/packages/flow/tests/fixtures/search_dialog/de_marsh/events.jsonl
new file mode 100644
index 0000000000..36700f0ab3
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/de_marsh/events.jsonl
@@ -0,0 +1,4 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.614}
+{"i": 1, "kind": "type", "text": "Elena Marsh", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.306}
+{"i": 2, "kind": "click", "x": 113, "y": 199, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.925}
+{"i": 3, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 3.491}
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0000_after.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0000_before.png
new file mode 100644
index 0000000000..28a1f29e73
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0001_after.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0001_before.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0002_after.png
new file mode 100644
index 0000000000..f82779f04c
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0002_before.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0003_after.png
new file mode 100644
index 0000000000..dd60851a8e
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0003_before.png
new file mode 100644
index 0000000000..f82779f04c
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh/meta.json b/packages/flow/tests/fixtures/search_dialog/de_marsh/meta.json
new file mode 100644
index 0000000000..c52308969f
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/de_marsh/meta.json
@@ -0,0 +1,14 @@
+{
+ "id": "a066387dda8b41d89f77d476794770e6",
+ "created_at": "2026-11-03T19:49:07.914570+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Elena Marsh"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/events.jsonl b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/events.jsonl
new file mode 100644
index 0000000000..7999f509d4
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/events.jsonl
@@ -0,0 +1,4 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 0.709}
+{"i": 1, "kind": "type", "text": "Elena Marsh", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.348}
+{"i": 2, "kind": "click", "x": 113, "y": 199, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.827}
+{"i": 3, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.35}
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0000_after.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0000_before.png
new file mode 100644
index 0000000000..28a1f29e73
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0001_after.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0001_before.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0002_after.png
new file mode 100644
index 0000000000..f82779f04c
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0002_before.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0003_after.png
new file mode 100644
index 0000000000..dd60851a8e
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0003_before.png
new file mode 100644
index 0000000000..f82779f04c
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_marsh_again/meta.json b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/meta.json
new file mode 100644
index 0000000000..e6d406c71e
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/de_marsh_again/meta.json
@@ -0,0 +1,14 @@
+{
+ "id": "19707a34f5a549e9a7b2c308e7ea1626",
+ "created_at": "2026-11-03T19:49:39.317129+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Elena Marsh"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/events.jsonl b/packages/flow/tests/fixtures/search_dialog/de_weber/events.jsonl
new file mode 100644
index 0000000000..8ac8d54dd7
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/de_weber/events.jsonl
@@ -0,0 +1,4 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 0.609}
+{"i": 1, "kind": "type", "text": "Jonas Weber", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.114}
+{"i": 2, "kind": "click", "x": 113, "y": 199, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.638}
+{"i": 3, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.205}
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0000_after.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0000_before.png
new file mode 100644
index 0000000000..28a1f29e73
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0001_after.png
new file mode 100644
index 0000000000..7650128874
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0001_before.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0002_after.png
new file mode 100644
index 0000000000..0640ff4ddc
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0002_before.png
new file mode 100644
index 0000000000..7650128874
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0003_after.png
new file mode 100644
index 0000000000..41de7e136c
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0003_before.png
new file mode 100644
index 0000000000..0640ff4ddc
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/de_weber/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/de_weber/meta.json b/packages/flow/tests/fixtures/search_dialog/de_weber/meta.json
new file mode 100644
index 0000000000..e87b15bc99
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/de_weber/meta.json
@@ -0,0 +1,14 @@
+{
+ "id": "e9cfdb2df4de4920a277712353a9ba99",
+ "created_at": "2026-11-03T19:49:23.100321+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Jonas Weber"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/events.jsonl b/packages/flow/tests/fixtures/search_dialog/en_weber/events.jsonl
new file mode 100644
index 0000000000..43f622a355
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/en_weber/events.jsonl
@@ -0,0 +1,4 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 0.825}
+{"i": 1, "kind": "type", "text": "Jonas Weber", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.445}
+{"i": 2, "kind": "click", "x": 113, "y": 199, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.018}
+{"i": 3, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.753}
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0000_after.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0000_before.png
new file mode 100644
index 0000000000..28a1f29e73
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0001_after.png
new file mode 100644
index 0000000000..7650128874
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0001_before.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0002_after.png
new file mode 100644
index 0000000000..1e537b7b81
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0002_before.png
new file mode 100644
index 0000000000..7650128874
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0003_after.png
new file mode 100644
index 0000000000..41de7e136c
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0003_before.png
new file mode 100644
index 0000000000..1e537b7b81
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/en_weber/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/en_weber/meta.json b/packages/flow/tests/fixtures/search_dialog/en_weber/meta.json
new file mode 100644
index 0000000000..5327d2a0b5
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/en_weber/meta.json
@@ -0,0 +1,14 @@
+{
+ "id": "d626ebf6d75e4197b3314e871e122f9a",
+ "created_at": "2026-11-03T19:50:55.450027+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Jonas Weber"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/events.jsonl b/packages/flow/tests/fixtures/search_dialog/names_marsh/events.jsonl
new file mode 100644
index 0000000000..5e06dcd6b6
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/names_marsh/events.jsonl
@@ -0,0 +1,4 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.128}
+{"i": 1, "kind": "type", "text": "Elena Marsh", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.891}
+{"i": 2, "kind": "click", "x": 113, "y": 199, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.644}
+{"i": 3, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 3.457}
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0000_after.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0000_before.png
new file mode 100644
index 0000000000..28a1f29e73
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0001_after.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0001_before.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0002_after.png
new file mode 100644
index 0000000000..557ffac7cf
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0002_before.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0003_after.png
new file mode 100644
index 0000000000..dd60851a8e
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0003_before.png
new file mode 100644
index 0000000000..557ffac7cf
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/names_marsh/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/names_marsh/meta.json b/packages/flow/tests/fixtures/search_dialog/names_marsh/meta.json
new file mode 100644
index 0000000000..813ac4aad2
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/names_marsh/meta.json
@@ -0,0 +1,14 @@
+{
+ "id": "578d12cf67274740be88c2e3665fe83c",
+ "created_at": "2026-11-03T19:50:37.782601+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Elena Marsh"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/events.jsonl b/packages/flow/tests/fixtures/search_dialog/question_marsh/events.jsonl
new file mode 100644
index 0000000000..a9434ed32e
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/question_marsh/events.jsonl
@@ -0,0 +1,4 @@
+{"i": 0, "kind": "click", "x": 256, "y": 122, "structural": {"selector": "#name", "role": "textbox", "name": "Name"}, "structured_identity": "Name", "structured_row": false, "section": {"kind": "heading", "text": "New record"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 1.852}
+{"i": 1, "kind": "type", "text": "Elena Marsh", "field_label": "Name", "param": "name", "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.344}
+{"i": 2, "kind": "click", "x": 113, "y": 199, "structural": {"selector": "#create", "role": "button", "name": "Create record"}, "section": {"kind": "heading", "text": "New record"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 2.784}
+{"i": 3, "kind": "click", "x": 459, "y": 411, "structural": {"selector": "#confirm", "role": "button", "name": "Confirm"}, "section": {"kind": "dialog", "text": "Dublettensuche"}, "submit_binding": {"v": 1, "tag": "button", "type": "button", "labelled": true, "in_row": false, "fixed": false, "fields": 1, "form_owner": "none", "submit_role": "detached", "scope_kind": "container", "scope_path": "div>div>button", "fields_in_scope": true, "field_local": true, "order": "after"}, "viewport_before": [1280, 800], "viewport_after": [1280, 800], "url_before": "http://records.test/new", "title_before": "Records", "pages_before": 1, "url_after": "http://records.test/new", "title_after": "Records", "pages_after": 1, "t": 3.354}
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0000_after.png b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0000_after.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0000_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0000_before.png b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0000_before.png
new file mode 100644
index 0000000000..28a1f29e73
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0000_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0001_after.png b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0001_after.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0001_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0001_before.png b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0001_before.png
new file mode 100644
index 0000000000..2e0503a227
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0001_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0002_after.png b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0002_after.png
new file mode 100644
index 0000000000..0e411aaca0
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0002_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0002_before.png b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0002_before.png
new file mode 100644
index 0000000000..e9df6e500d
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0002_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0003_after.png b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0003_after.png
new file mode 100644
index 0000000000..dd60851a8e
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0003_after.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0003_before.png b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0003_before.png
new file mode 100644
index 0000000000..0e411aaca0
Binary files /dev/null and b/packages/flow/tests/fixtures/search_dialog/question_marsh/frames/0003_before.png differ
diff --git a/packages/flow/tests/fixtures/search_dialog/question_marsh/meta.json b/packages/flow/tests/fixtures/search_dialog/question_marsh/meta.json
new file mode 100644
index 0000000000..da846f6a34
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/question_marsh/meta.json
@@ -0,0 +1,14 @@
+{
+ "id": "d54cae7430f54f9382c607078aee5c49",
+ "created_at": "2026-11-03T20:42:59.851089+00:00",
+ "viewport": [
+ 1280,
+ 800
+ ],
+ "coordinate_space": "page",
+ "app_url": "http://records.test/",
+ "params": {
+ "name": "Elena Marsh"
+ },
+ "secret_params": []
+}
\ No newline at end of file
diff --git a/packages/flow/tests/fixtures/search_dialog/record.py b/packages/flow/tests/fixtures/search_dialog/record.py
new file mode 100644
index 0000000000..1e9e440266
--- /dev/null
+++ b/packages/flow/tests/fixtures/search_dialog/record.py
@@ -0,0 +1,92 @@
+"""Record the search-dialog fixtures with the real recorder and Chromium.
+
+Run from ``packages/flow``: ``python tests/fixtures/search_dialog/record.py``.
+Each recording types a name, clicks "Create record" (which opens a duplicate
+search dialog) and clicks the dialog's "Confirm". The frames are what the
+browser showed; the tests read them with the real OCR.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+from pathlib import Path
+from urllib.parse import quote
+
+from openadapt_flow.backends.playwright_backend import PlaywrightBackend
+from openadapt_flow.recorder import Recorder
+
+HERE = Path(__file__).resolve().parent
+#: recording -> (statement the dialog shows, name typed)
+RECORDINGS = {
+ "de_marsh": ("Keine Treffer gefunden.", "Elena Marsh"),
+ "de_weber": ("Keine Treffer gefunden.", "Jonas Weber"),
+ "de_marsh_again": ("Keine Treffer gefunden.", "Elena Marsh"),
+ "archive_marsh": ("Keine Treffer im Archiv {archive}", "Elena Marsh"),
+ "archive_weber": ("Keine Treffer im Archiv {archive}", "Jonas Weber"),
+ "names_marsh": ("Keine Treffer für {name}", "Elena Marsh"),
+ "en_weber": ("No results found", "Jonas Weber"),
+ "question_marsh": ("Wirklich speichern?", "Elena Marsh"),
+}
+#: Recordings that also type a comment after the name (the search is the same).
+COMMENTS = {
+ "comment_marsh_first": (
+ "Keine Treffer im Archiv {archive}",
+ "Elena Marsh",
+ "Erster Anruf",
+ ),
+ "comment_marsh_second": (
+ "Keine Treffer im Archiv {archive}",
+ "Elena Marsh",
+ "Rueckruf morgen",
+ ),
+}
+
+
+def record(name: str, statement: str, typed: str, comment: str = "") -> None:
+ out = HERE / name
+ if out.exists():
+ shutil.rmtree(out)
+ url = (HERE / "app.html").as_uri() + "?s=" + quote(statement)
+ if comment:
+ url += "&c=1"
+ backend, close = PlaywrightBackend.launch(url, headless=True)
+ try:
+ page = backend.page
+
+ def center(selector: str) -> tuple[int, int]:
+ box = page.locator(selector).first.bounding_box()
+ assert box is not None
+ return int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2)
+
+ recorder = Recorder(backend, out, app_url="http://records.test/")
+ recorder.click(*center("#name"))
+ recorder.type_text(typed, param="name")
+ if comment:
+ recorder.click(*center("#comment"))
+ recorder.type_text(comment, param="comment")
+ recorder.click(*center("#create"))
+ recorder.click(*center("#confirm"))
+ recorder.finish()
+ finally:
+ close()
+ # The page was opened from a local file; keep the machine's path out.
+ events = out / "events.jsonl"
+ events.write_text(
+ "".join(
+ json.dumps(
+ {
+ k: ("http://records.test/new" if k.startswith("url_") else v)
+ for k, v in json.loads(line).items()
+ }
+ )
+ + "\n"
+ for line in events.read_text().splitlines()
+ )
+ )
+
+
+if __name__ == "__main__":
+ for key in sys.argv[1:] or [*RECORDINGS, *COMMENTS]:
+ record(key, *(RECORDINGS.get(key) or COMMENTS[key]))
diff --git a/packages/flow/tests/test_admit_pipeline.py b/packages/flow/tests/test_admit_pipeline.py
index d0b1425264..834ad23e03 100644
--- a/packages/flow/tests/test_admit_pipeline.py
+++ b/packages/flow/tests/test_admit_pipeline.py
@@ -379,7 +379,9 @@ def test_refused_induction_stops_without_a_bundle(tmp_path, monkeypatch) -> None
monkeypatch.setattr(
induction,
"induce_program",
- lambda traces: SimpleNamespace(certified=False, workflow=None, record_next=[]),
+ lambda traces, **_kw: SimpleNamespace(
+ certified=False, workflow=None, record_next=[]
+ ),
)
pending = ap.prepare_admission(
[_recording(tmp_path, "rec1"), _recording(tmp_path, "rec2")],
diff --git a/packages/flow/tests/test_empty_result_guard_evidence.py b/packages/flow/tests/test_empty_result_guard_evidence.py
new file mode 100644
index 0000000000..558a1ee8fe
--- /dev/null
+++ b/packages/flow/tests/test_empty_result_guard_evidence.py
@@ -0,0 +1,358 @@
+"""A dialog's empty-result statement in any language, mined from evidence.
+
+The vocabulary in ``compiler.empty_result_guard`` is English. A statement in
+other words ("Keine Treffer gefunden.") is mined only when two or more
+demonstrations with different searches show the same line: record data
+changes with the search, the dialog's own words do not. A guard the
+demonstrations disagree on is not copied from the first one.
+
+The recordings in ``tests/fixtures/search_dialog`` were made with the real
+recorder in Chromium (``record.py`` there) and are read with the real OCR.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from openadapt_flow.compiler.empty_result_guard import (
+ dialog_statement,
+ looks_like_data,
+)
+from openadapt_flow.compiler.induction import InductionResult, induce_program
+from openadapt_flow.ir import PostconditionKind, PredicateKind, Step, Workflow
+from openadapt_flow.vision.ocr import OcrLine, ocr
+
+FIXTURES = Path(__file__).parent / "fixtures" / "search_dialog"
+GERMAN = "Keine Treffer gefunden."
+RECORDINGS = (
+ "de_marsh",
+ "de_weber",
+ "de_marsh_again",
+ "archive_marsh",
+ "archive_weber",
+ "names_marsh",
+ "en_weber",
+ "comment_marsh_first",
+ "comment_marsh_second",
+)
+
+
+@pytest.fixture(scope="module")
+def compiled(
+ tmp_path_factory: pytest.TempPathFactory,
+) -> dict[str, tuple[Workflow, list[dict[str, Any]]]]:
+ """Each recording compiled once: its workflow and its compile notes."""
+
+ from openadapt_flow.compiler import compile_recording
+ from openadapt_flow.compiler.fixup_passes import read_compile_notes
+
+ root = tmp_path_factory.mktemp("search-dialog")
+ out = {}
+ for name in RECORDINGS:
+ bundle = root / name
+ workflow = compile_recording(FIXTURES / name, bundle, name=name)
+ out[name] = (workflow, read_compile_notes(bundle))
+ return out
+
+
+def _induce(compiled: dict, *names: str) -> InductionResult:
+ return induce_program(
+ [compiled[name][0] for name in names],
+ compile_notes=[compiled[name][1] for name in names],
+ )
+
+
+def _steps(result: InductionResult) -> tuple[Step, Step]:
+ assert result.workflow is not None and result.workflow.program is not None
+ states = result.workflow.program.states
+ steps = [state.step for state in states.values() if state.step is not None]
+ opener = next(s for s in steps if s.intent and "Create" in s.intent)
+ confirm = next(s for s in steps if s.intent and "Confirm" in s.intent)
+ return opener, confirm
+
+
+def _exact_texts(step: Step) -> list[str]:
+ return [pc.text or "" for pc in step.expect if pc.exact]
+
+
+def _frames(name: str) -> tuple[list[OcrLine], list[OcrLine], list[OcrLine]]:
+ frames = FIXTURES / name / "frames"
+ return (
+ ocr((frames / "0002_before.png").read_bytes()),
+ ocr((frames / "0002_after.png").read_bytes()),
+ ocr((frames / "0003_before.png").read_bytes()),
+ )
+
+
+def _confirm_point(name: str) -> tuple[int, int]:
+ """Where the recording clicked Confirm (its last event)."""
+
+ lines = (FIXTURES / name / "events.jsonl").read_text().splitlines()
+ event = json.loads(lines[-1])
+ return int(event["x"]), int(event["y"])
+
+
+CONFIRM = _confirm_point("de_marsh")
+
+
+# -- the candidate line, one screen at a time ------------------------------------
+def test_the_dialogs_one_line_is_found_in_any_language() -> None:
+ line, why = dialog_statement(
+ *_frames("de_marsh"), confirm_point=CONFIRM, title="Dublettensuche"
+ )
+ assert (line, why) == (GERMAN, "")
+
+
+def test_an_unnamed_dialog_gives_no_line() -> None:
+ """Without the dialog's name, its title could be the line."""
+
+ line, why = dialog_statement(
+ *_frames("de_marsh"), confirm_point=CONFIRM, title=None
+ )
+ assert line is None and "title" in why
+
+
+def test_the_title_and_button_row_are_not_the_line() -> None:
+ # Named something else, the title is a second line: nothing is chosen.
+ line, why = dialog_statement(
+ *_frames("de_marsh"), confirm_point=CONFIRM, title="Another title"
+ )
+ assert line is None and "2 lines" in why
+
+
+def test_a_line_with_the_typed_value_is_refused() -> None:
+ from openadapt_flow.compiler.compile import _contains_excluded
+
+ line, why = dialog_statement(
+ *_frames("names_marsh"),
+ confirm_point=CONFIRM,
+ title="Dublettensuche",
+ reject=lambda text: _contains_excluded(text, ("Elena Marsh",)),
+ )
+ assert line is None and "run value" in why
+
+
+def test_numbers_and_addresses_are_data() -> None:
+ assert not looks_like_data("Keine Treffer gefunden.")
+ assert not looks_like_data("0 Treffer")
+ assert looks_like_data("3 Treffer")
+ assert looks_like_data("Geboren 1958-03-27")
+ assert looks_like_data("Kontakt: a@b.example")
+
+
+# -- one demonstration ----------------------------------------------------------
+def test_one_demonstration_keeps_the_line_only_as_a_candidate(
+ compiled: dict,
+) -> None:
+ workflow, notes = compiled["de_marsh"]
+ assert workflow.steps[3].guard is None
+ assert not any(pc.exact for step in workflow.steps for pc in step.expect)
+ (missed,) = [n for n in notes if n["change"] == "no_empty_result_guard"]
+ assert missed["dialog_line"] == GERMAN
+ assert missed["step"] == "step_002" and missed["confirm_step"] == "step_003"
+
+
+def test_one_demonstration_alone_is_not_mined(compiled: dict) -> None:
+ result = _induce(compiled, "de_marsh")
+ _opener, confirm = _steps(result)
+ assert confirm.guard is None
+ (note,) = result.guard_notes
+ assert note["change"] == "no_empty_result_guard"
+ assert "one demonstration" in note["warning"]
+
+
+# -- evidence across demonstrations ----------------------------------------------
+def test_the_same_line_across_different_searches_is_mined(compiled: dict) -> None:
+ result = _induce(compiled, "de_marsh", "de_weber")
+ assert result.certified, result.uncertainties
+ opener, confirm = _steps(result)
+ assert _exact_texts(opener) == [GERMAN]
+ assert confirm.guard is not None and confirm.guard.on_unmet == "halt"
+ assert confirm.guard.predicate.kind is PredicateKind.TEXT_PRESENT
+ assert confirm.guard.predicate.text == GERMAN and confirm.guard.predicate.exact
+ (note,) = result.guard_notes
+ assert note["change"] == "empty_result_guard" and "typed" in note["evidence"]
+
+
+def test_recording_directories_carry_their_candidates() -> None:
+ result = induce_program([FIXTURES / "de_marsh", FIXTURES / "de_weber"])
+ _opener, confirm = _steps(result)
+ assert confirm.guard is not None and confirm.guard.predicate.text == GERMAN
+
+
+def test_a_line_that_varies_with_the_data_is_not_mined(compiled: dict) -> None:
+ """ "Keine Treffer im Archiv Nord" / "... West": no value, no number, but
+ it changes with the record searched, so it is data."""
+
+ result = _induce(compiled, "archive_marsh", "archive_weber")
+ opener, confirm = _steps(result)
+ assert confirm.guard is None and _exact_texts(opener) == []
+ (note,) = result.guard_notes
+ assert "differs between demonstrations" in note["warning"]
+
+
+def test_a_line_with_a_parameter_is_not_mined(compiled: dict) -> None:
+ _workflow, notes = compiled["names_marsh"]
+ (missed,) = [n for n in notes if n["change"] == "no_empty_result_guard"]
+ assert "dialog_line" not in missed
+ assert "run value" in missed["dialog_line_note"]
+ result = _induce(compiled, "names_marsh", "de_weber")
+ _opener, confirm = _steps(result)
+ assert confirm.guard is None
+ assert "not every demonstration" in result.guard_notes[0]["warning"]
+
+
+def test_the_same_search_twice_proves_nothing(compiled: dict) -> None:
+ result = _induce(compiled, "de_marsh", "de_marsh_again")
+ _opener, confirm = _steps(result)
+ assert confirm.guard is None
+ assert "typed before the dialog is the same" in result.guard_notes[0]["warning"]
+
+
+def test_a_later_field_that_differs_does_not_prove_the_search_differed(
+ compiled: dict,
+) -> None:
+ """The same name both times, then a different comment: the archive line
+ depends on the name, which did not change, so it proves nothing."""
+
+ for name in ("comment_marsh_first", "comment_marsh_second"):
+ (missed,) = [
+ n for n in compiled[name][1] if n["change"] == "no_empty_result_guard"
+ ]
+ # Both compiles kept the line: the refusal is induction's.
+ assert missed["dialog_line"] == "Keine Treffer im Archiv Nord"
+ result = _induce(compiled, "comment_marsh_first", "comment_marsh_second")
+ assert result.certified, result.uncertainties
+ opener, confirm = _steps(result)
+ assert confirm.guard is None and _exact_texts(opener) == []
+ (note,) = result.guard_notes
+ assert note["change"] == "no_empty_result_guard"
+ assert "typed before the dialog is the same" in note["warning"]
+ assert note["step"].startswith("step_") and note["confirm_step"].startswith("step_")
+
+
+def test_a_guard_the_demonstrations_disagree_on_refuses_induction(
+ compiled: dict,
+) -> None:
+ """The English recording's compile mined "No results found"; the German
+ one mined nothing. Copying the first one's guard would pick one dialog,
+ dropping it would click through a listed match: induction refuses."""
+
+ workflow, _notes = compiled["en_weber"]
+ assert workflow.steps[3].guard is not None
+ for order in (("en_weber", "de_marsh"), ("de_marsh", "en_weber")):
+ result = _induce(compiled, *order)
+ assert not result.certified and result.workflow is None
+ assert [u.kind for u in result.uncertainties] == ["ambiguous_guard"]
+ assert result.uncertainties[0].consequential
+ (note,) = result.guard_notes
+ assert note["change"] == "no_empty_result_guard"
+ assert "disagree" in note["warning"]
+
+
+def test_a_question_is_not_a_search_result() -> None:
+ line, why = dialog_statement(
+ *_frames("question_marsh"),
+ confirm_point=_confirm_point("question_marsh"),
+ title="Dublettensuche",
+ )
+ assert line is None and "question" in why
+
+
+def test_a_disagreeing_guard_off_the_straight_path_refuses() -> None:
+ """An optional or loop column copies one trace's step: its mined check
+ must be the same at that place in every trace and row."""
+
+ from openadapt_flow.compiler.empty_result_guard import exact_checks
+ from openadapt_flow.compiler.induction import (
+ _Column,
+ _refuse_unreconciled_guards,
+ _SingleTok,
+ )
+ from openadapt_flow.ir import ActionKind
+
+ def confirm(text: str | None) -> Step:
+ step = Step(id="step_003", intent="click 'Confirm'", action=ActionKind.CLICK)
+ if text:
+ step.guard = exact_checks(text)[1]
+ return step
+
+ for texts, refused in (
+ (("Keine Treffer gefunden.", None), True),
+ (("Keine Treffer gefunden.", "No results found"), True),
+ (("Keine Treffer gefunden.", "Keine Treffer gefunden."), False),
+ ((None, None), False),
+ ):
+ col = _Column(("click", "confirm:select:button"), "single")
+ for t, text in enumerate(texts):
+ col.tokens[t] = _SingleTok(confirm(text))
+ result = InductionResult(n_traces=2)
+ _refuse_unreconciled_guards([col], set(), result)
+ assert bool(result.uncertainties) is refused, texts
+
+ # A Confirm off the straight path with no check is noted once.
+ col = _Column(("click", "confirm:select:button"), "single")
+ col.tokens[0] = _SingleTok(confirm(None))
+ col.tokens[1] = _SingleTok(confirm(None))
+ result = InductionResult(n_traces=2)
+ pairs = [{("step_002", "step_003")}, {("step_002", "step_003")}]
+ _refuse_unreconciled_guards([col], set(), result, pairs)
+ assert not result.uncertainties
+ (note,) = result.guard_notes
+ assert note["change"] == "no_empty_result_guard"
+ assert (note["step"], note["confirm_step"]) == ("step_002", "step_003")
+ assert "straight path" in note["warning"]
+
+ # The opener's exact postcondition is compared the same way.
+ opener = Step(id="step_002", intent="click 'Create'", action=ActionKind.CLICK)
+ col = _Column(("click", "select:button"), "single")
+ col.tokens[0] = _SingleTok(opener)
+ col.tokens[1] = _SingleTok(
+ opener.model_copy(update={"expect": [exact_checks("No results found")[0]]})
+ )
+ result = InductionResult(n_traces=2)
+ _refuse_unreconciled_guards([col], set(), result)
+ assert [u.kind for u in result.uncertainties] == ["ambiguous_guard"]
+
+
+def test_a_guard_every_demonstration_mined_is_kept(compiled: dict) -> None:
+ result = _induce(compiled, "en_weber", "en_weber")
+ opener, confirm = _steps(result)
+ assert confirm.guard is not None
+ assert confirm.guard.predicate.text == "No results found"
+ assert _exact_texts(opener) == ["No results found"]
+ assert result.guard_notes[0]["change"] == "empty_result_guard"
+
+
+# -- qualification ----------------------------------------------------------------
+def test_qualification_needs_a_run_that_halts_at_the_mined_line(
+ compiled: dict,
+) -> None:
+ """On the induced program itself (a graph, no linear steps)."""
+
+ from openadapt_flow.qualify.campaign import business_rule_problems, rule_halt_steps
+ from openadapt_flow.qualify.config import BusinessRuleCase, VerifyConfig
+
+ result = _induce(compiled, "de_marsh", "de_weber")
+ workflow = result.workflow
+ assert workflow is not None and workflow.steps == []
+ opener, confirm = _steps(result)
+ config = VerifyConfig.model_construct(business_rules=[])
+ (problem,) = business_rule_problems(workflow, config, confirm, {})
+ assert "business_rules case" in problem and confirm.id in problem
+ rule = BusinessRuleCase.model_construct(
+ id="duplicate", params={}, halt_at="empty_result_guard"
+ )
+ assert rule_halt_steps(workflow, rule, confirm.id) == {opener.id, confirm.id}
+ config = VerifyConfig.model_construct(business_rules=[rule])
+ assert business_rule_problems(workflow, config, confirm, {}) == []
+
+
+def test_the_mined_check_is_a_text_check(compiled: dict) -> None:
+ opener, _confirm = _steps(_induce(compiled, "de_marsh", "de_weber"))
+ (pc,) = [pc for pc in opener.expect if pc.exact]
+ assert pc.kind is PostconditionKind.TEXT_PRESENT