diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index 53f64903..98bcc887 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -41,6 +41,7 @@ import math import os import re +import struct import time import uuid from copy import deepcopy @@ -204,6 +205,26 @@ # the recorded region (real apps re-layout by a few pixels between runs), # and the minimum template-match score to accept it. PC_TEMPLATE_SEARCH_PAD = 80 + + +def _frame_viewport(frame_png: bytes) -> tuple[int, int]: + """Return the viewport of THIS frame, read from the frame itself. + + ``self.backend.viewport`` is a live read. Pairing it with an + already-captured frame lets a resize or move between capture and use + reinterpret that frame under geometry it never had, which can place a + coordinate where the operator never demonstrated one. A PNG carries its own + dimensions in the IHDR chunk, so the frame can always answer for itself. + """ + + if len(frame_png) < 24 or not frame_png.startswith(b"\x89PNG\r\n\x1a\n"): + raise ValueError("frame viewport requires valid PNG bytes") + width, height = struct.unpack(">II", frame_png[16:24]) + if width <= 0 or height <= 0: + raise ValueError("frame viewport must be positive") + return int(width), int(height) + + PC_TEMPLATE_THRESHOLD = 0.9 # REGION_STABLE asserts recorded structure, not palette. The ordinary # grayscale template matcher remains the first (stricter) check; this edge-map @@ -8790,7 +8811,7 @@ def _resolve_step( self.grounder if allow_grounder else None, step.intent, template_png=template_png, - viewport=self.backend.viewport, + viewport=_frame_viewport(screen_png), structural=structural, allow_target_ocr=allow_target_ocr, ) @@ -10822,7 +10843,7 @@ def _handle_interstitials( None, # NEVER ground a dismissal: stay model-free it.name, template_png=template_png, - viewport=self.backend.viewport, + viewport=_frame_viewport(before_png), structural=structural, ) if res is None: @@ -11152,7 +11173,7 @@ def _predicate_holds( frame_png, params, vision=self.vision, - viewport=self.backend.viewport, + viewport=_frame_viewport(frame_png), asset_loader=lambda rel: self._asset_bytes( bundle_dir, rel, @@ -12553,7 +12574,7 @@ def _implicit_scroll_target_ready( None, # scroll readiness must remain deterministic and model-free step.intent, template_png=template_png, - viewport=self.backend.viewport, + viewport=_frame_viewport(frame_png), structural=structural, ) except OcrResolutionRefused: @@ -13044,7 +13065,7 @@ def _postcondition_passes( template_png = self._postcondition_template(pc, bundle_dir) if template_png is not None: search = pad_region( - region, PC_TEMPLATE_SEARCH_PAD, self.backend.viewport + region, PC_TEMPLATE_SEARCH_PAD, _frame_viewport(frame_png) ) match = self.vision.find_template( frame_png,