From 0af1202ddce3979d9c3f889a4cb0cbeea1f38512 Mon Sep 17 00:00:00 2001
From: Richard Abrich <richard.abrich@mldsai.com>
Date: Wed, 26 Aug 2026 14:20:12 -0400
Subject: [PATCH 1/2] fix(runtime): read each frame's viewport from the frame,
 not live from the backend

The replayer resolved targets and clamped postcondition search regions against
an already-captured frame while reading self.backend.viewport live at call
time. If the window moved, resized, or the display rescaled between the
capture and the use, the frame was interpreted under geometry it never had --
placing a coordinate where the operator never demonstrated one, with the screen
still looking correct. That is a silent wrong action produced by the runtime
itself.

A PNG carries its own dimensions in its IHDR chunk, so a frame can always
answer for itself. _frame_viewport() reads them and validates the signature.

Five sites had a captured frame in scope and are rebound to it. The four
remaining self.backend.viewport reads have no frame in scope and are correct
as live reads.

Found while recovering abandoned work (see issue #402), whose 4,602-line
backend refactor contains the same idea; this extracts it alone.
---
 openadapt_flow/runtime/replayer.py | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py
index 53f64903..d582210b 100644
--- a/openadapt_flow/runtime/replayer.py
+++ b/openadapt_flow/runtime/replayer.py
@@ -41,6 +41,7 @@
 import math
 import os
 import re
+import struct
 import time
 import uuid
 from copy import deepcopy
@@ -204,6 +205,25 @@
 # the recorded region (real apps re-layout by a few pixels between runs),
 # and the minimum template-match score to accept it.
 PC_TEMPLATE_SEARCH_PAD = 80
+
+def _frame_viewport(frame_png: bytes) -> tuple[int, int]:
+    """Return the viewport of THIS frame, read from the frame itself.
+
+    ``self.backend.viewport`` is a live read. Pairing it with an
+    already-captured frame lets a resize or move between capture and use
+    reinterpret that frame under geometry it never had, which can place a
+    coordinate where the operator never demonstrated one. A PNG carries its own
+    dimensions in the IHDR chunk, so the frame can always answer for itself.
+    """
+
+    if len(frame_png) < 24 or not frame_png.startswith(b"\x89PNG\r\n\x1a\n"):
+        raise ValueError("frame viewport requires valid PNG bytes")
+    width, height = struct.unpack(">II", frame_png[16:24])
+    if width <= 0 or height <= 0:
+        raise ValueError("frame viewport must be positive")
+    return int(width), int(height)
+
+
 PC_TEMPLATE_THRESHOLD = 0.9
 # REGION_STABLE asserts recorded structure, not palette.  The ordinary
 # grayscale template matcher remains the first (stricter) check; this edge-map
@@ -8790,7 +8810,7 @@ def _resolve_step(
             self.grounder if allow_grounder else None,
             step.intent,
             template_png=template_png,
-            viewport=self.backend.viewport,
+            viewport=_frame_viewport(screen_png),
             structural=structural,
             allow_target_ocr=allow_target_ocr,
         )
@@ -10822,7 +10842,7 @@ def _handle_interstitials(
                     None,  # NEVER ground a dismissal: stay model-free
                     it.name,
                     template_png=template_png,
-                    viewport=self.backend.viewport,
+                    viewport=_frame_viewport(before_png),
                     structural=structural,
                 )
                 if res is None:
@@ -11152,7 +11172,7 @@ def _predicate_holds(
             frame_png,
             params,
             vision=self.vision,
-            viewport=self.backend.viewport,
+            viewport=_frame_viewport(frame_png),
             asset_loader=lambda rel: self._asset_bytes(
                 bundle_dir,
                 rel,
@@ -12553,7 +12573,7 @@ def _implicit_scroll_target_ready(
                 None,  # scroll readiness must remain deterministic and model-free
                 step.intent,
                 template_png=template_png,
-                viewport=self.backend.viewport,
+                viewport=_frame_viewport(frame_png),
                 structural=structural,
             )
         except OcrResolutionRefused:
@@ -13044,7 +13064,7 @@ def _postcondition_passes(
             template_png = self._postcondition_template(pc, bundle_dir)
             if template_png is not None:
                 search = pad_region(
-                    region, PC_TEMPLATE_SEARCH_PAD, self.backend.viewport
+                    region, PC_TEMPLATE_SEARCH_PAD, _frame_viewport(frame_png)
                 )
                 match = self.vision.find_template(
                     frame_png,

From 0d7fa6446b77ce506c2a5d2a74e968f08e43c99d Mon Sep 17 00:00:00 2001
From: Richard Abrich <richard.abrich@mldsai.com>
Date: Wed, 26 Aug 2026 15:15:17 -0400
Subject: [PATCH 2/2] style: blank line before the frame viewport helper (ruff
 format)

---
 openadapt_flow/runtime/replayer.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py
index d582210b..98bcc887 100644
--- a/openadapt_flow/runtime/replayer.py
+++ b/openadapt_flow/runtime/replayer.py
@@ -206,6 +206,7 @@
 # and the minimum template-match score to accept it.
 PC_TEMPLATE_SEARCH_PAD = 80
 
+
 def _frame_viewport(frame_png: bytes) -> tuple[int, int]:
     """Return the viewport of THIS frame, read from the frame itself.