feat: capture native structural observations

.github/workflows/production-qualification.yml

@@ -152,9 +152,15 @@ jobs:
           set -euo pipefail
           qualification_root="${RUNNER_TEMP}/capture-qualification-${GITHUB_RUN_ID}-${GITHUB_JOB}"
           qualification_python="${qualification_root}/bin/python"
+          candidate_wheels=(dist/*.whl)
+          if [ "${#candidate_wheels[@]}" -ne 1 ]; then
+            echo "Expected exactly one candidate wheel."
+            exit 1
+          fi
           uv venv --clear --python 3.12 "${qualification_root}"
           uv pip install --python "${qualification_python}" \
-            dist/*.whl pytest==9.1.1 pytest-timeout==2.4.0 pynput==1.8.2
+            "${candidate_wheels[0]}[linux]" \
+            pytest==9.1.1 pytest-timeout==2.4.0 pynput==1.8.2
           echo "QUALIFICATION_PYTHON=${qualification_python}" >> "${GITHUB_ENV}"
 
       - name: Require the reviewed external video tools
@@ -186,6 +192,7 @@ jobs:
             echo "::group::Linux trial ${trial}"
             "${QUALIFICATION_PYTHON}" -m pytest \
               "${GITHUB_WORKSPACE}/tests/test_performance.py" \
+              "${GITHUB_WORKSPACE}/tests/test_structural_observation.py" \
               -m slow -v --timeout=300 --import-mode=importlib \
               "--junitxml=${GITHUB_WORKSPACE}/evidence/trial-${trial}-linux.xml"
             echo "::endgroup::"
@@ -287,6 +294,7 @@ jobs:
             echo "::group::macOS trial ${trial}"
             "${QUALIFICATION_PYTHON}" -m pytest \
               "${GITHUB_WORKSPACE}/tests/test_performance.py" \
+              "${GITHUB_WORKSPACE}/tests/test_structural_observation.py" \
               "${GITHUB_WORKSPACE}/tests/test_window_capture.py" \
               -m slow -v --timeout=300 --import-mode=importlib \
               "--junitxml=${GITHUB_WORKSPACE}/evidence/trial-${trial}-macos.xml"
@@ -394,6 +402,7 @@ jobs:
               Write-Host "::group::Windows trial $trial"
               & $env:QUALIFICATION_PYTHON -m pytest `
                 "$env:GITHUB_WORKSPACE/tests/test_performance.py" `
+                "$env:GITHUB_WORKSPACE/tests/test_structural_observation.py" `
                 "$env:GITHUB_WORKSPACE/tests/test_window_capture.py" `
                 -m slow -v --timeout=300 --import-mode=importlib `
                 "--junitxml=$env:GITHUB_WORKSPACE/evidence/trial-$trial-windows.xml"

README.md

@@ -188,31 +188,39 @@ Desktop provisions and probes that exact closure.
 
 ## Native structural observations
 
-On Windows, the recorder can retain a versioned UI Automation observation
-beside the native action that produced it. When UIA exposes the information,
-the observation includes the target's AutomationId, role/control type, name,
-bounds, supported patterns, process/window identity, ancestry, and exact
-candidate cardinality within the top-level window. Unavailable values remain
-absent; capture never infers structural fields from coordinates or pixels.
-
-This evidence is stored in `recording.db`, exposed on raw events and processed
-`Action` objects, and remains optional so existing recordings and non-Windows
-hosts continue to load unchanged. It is enabled by default on Windows and can
-be disabled with
+The recorder can retain a versioned native accessibility observation beside
+each action. It uses Windows UI Automation, macOS Accessibility, or Linux
+AT-SPI. When the provider exposes them, Capture records the target identifier,
+role/control type, name, bounds, supported actions, process/window identity,
+and ancestry. Windows UIA also records exact candidate cardinality within the
+top-level window. Unavailable values remain absent. Capture never infers a
+structural field from coordinates or pixels.
+
+Capture stores this evidence in `recording.db` and exposes it on raw events and
+processed `Action` objects. The field is optional, so existing recordings still
+load unchanged. Native observation is enabled by default when its provider is
+available. Disable it with
 `Recorder(..., capture_structural_observations=False)`. Applications can inject
 another read-only observer through `Recorder(..., structural_observer=...)`
 using the public `StructuralObserver` protocol.
 
 Accessibility text remains inside the local raw-capture boundary and is bounded
 to 512 characters per field. Longer provider values are omitted rather than
 truncated, so partial text is never presented to the compiler as exact identity
-evidence. The versioned observation contract accepts namespaced provider IDs for
-future macOS Accessibility and Linux AT-SPI observers, but this package currently
-emits only `windows_uia` observations.
+evidence. Capture emits `windows_uia`, `macos_ax`, or `linux_atspi`. macOS
+requires Accessibility permission. Linux requires an available desktop AT-SPI
+bus and the system AT-SPI typelib. Install your distribution's PyGObject build
+packages and `gir1.2-atspi-2.0` first. The
+[PyGObject install guide](https://pygobject.gnome.org/getting_started.html)
+lists the current package names. Then install the `linux` extra:
 
-UIA describes the local Windows accessibility tree. It does not cross an
-RDP/Citrix pixel boundary into the remote application; those demonstrations
-retain window-scoped pixels and coordinates for Flow's remote visual compiler.
+```bash
+pip install "openadapt-capture[linux]"
+```
+
+The native provider describes the local accessibility tree. It can't see
+controls across an RDP or Citrix pixel boundary. Those demonstrations retain
+window-scoped pixels and coordinates for Flow's remote visual compiler.
 
 ## Window-scoped recording
 
@@ -388,8 +396,9 @@ session is required. See the
 
 - Native recording requires a visible user session plus the operating system's
   screen-recording and input-monitoring permissions.
-- Native Windows capture retains UIA evidence when the application exposes it;
-  opaque remote applications still require Flow's visual and OCR bindings.
+- Native Windows, macOS, and Linux capture retain accessibility evidence when
+  the application and local provider expose it. Opaque remote applications
+  still require Flow's visual and OCR bindings.
 - The Flow adapter compiles left and right clicks, left-button drags, typed
   text, named keys, modifier chords, and scrolling. It rejects unsupported
   input such as middle clicks, non-left-button drags, malformed shortcuts, and

docs/DESIGN.md

@@ -29,9 +29,9 @@ operation exports it.
 | Browser | `openadapt-flow` owns the supported Playwright recorder. The Chrome extension and bridge in this repository are source-only development prototypes and are excluded from Capture release artifacts. |
 
 Capture supports native input observation on macOS, Windows, and X11 Linux.
-Windows can also retain UI Automation evidence at action time. The structural
-schema permits another injected provider, but the package does not currently
-ship macOS Accessibility or Linux AT-SPI structural observers.
+It retains action-time structure through Windows UI Automation, macOS
+Accessibility, and Linux AT-SPI when the local provider is available. The
+public structural protocol also accepts an injected read-only provider.
 
 ## Session pipeline
 
@@ -159,14 +159,20 @@ schema can retain:
 - bounded ancestry
 - exact candidate cardinality and its matching fields
 
-The package currently creates a Windows UIA observer. A missing optional field
-stays missing. Capture does not infer an accessibility value from a screenshot,
-coordinate, or neighboring control. Provider text has strict length and depth
-bounds. A transient provider failure omits the optional observation without
-corrupting the screen and input evidence.
+The package creates a Windows UIA, macOS Accessibility, or Linux AT-SPI
+observer for the current platform. A missing optional field stays missing.
+Capture doesn't infer an accessibility value from a screenshot, coordinate, or
+neighboring control. Provider text has strict length and depth bounds. A
+transient provider failure omits the optional observation without corrupting
+the screen and input evidence.
 
-UIA describes the local accessibility tree. It does not describe controls
-inside an RDP or Citrix pixel stream.
+The Linux provider uses the GObject AT-SPI binding. The `linux` package extra
+installs the reviewed `PyGObject>=3.46,<3.50` range, which supports systems with
+GLib 2.64 or newer. The host supplies the PyGObject build packages, the AT-SPI
+typelib/runtime, and an interactive desktop accessibility bus.
+
+The native provider describes the local accessibility tree. It does not
+describe controls inside an RDP or Citrix pixel stream.
 
 ## Video and frame timing
 

openadapt_capture/structural.py

@@ -8,15 +8,14 @@
 from __future__ import annotations
 
 import logging
+import math
 import sys
 from dataclasses import dataclass
 from typing import Annotated, Literal, Protocol, runtime_checkable
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, model_validator
 
-STRUCTURAL_OBSERVATION_SCHEMA_VERSION = (
-    "openadapt.capture.structural-observation/v1"
-)
+STRUCTURAL_OBSERVATION_SCHEMA_VERSION = "openadapt.capture.structural-observation/v1"
 MAX_STRUCTURAL_TEXT_LENGTH = 512
 MAX_STRUCTURAL_ANCESTRY_DEPTH = 32
 
@@ -40,6 +39,16 @@ class StructuralBounds(BaseModel):
     right: float
     bottom: float
 
+    @model_validator(mode="after")
+    def validate_bounds(self) -> "StructuralBounds":
+        """Reject a non-finite or inverted provider rectangle."""
+        values = (self.left, self.top, self.right, self.bottom)
+        if not all(math.isfinite(value) for value in values):
+            raise ValueError("structural bounds must be finite")
+        if self.right < self.left or self.bottom < self.top:
+            raise ValueError("structural bounds must not be inverted")
+        return self
+
 
 class StructuralElement(BaseModel):
     """Stable and semantic fields exposed for one accessibility element."""
@@ -102,23 +111,21 @@ class StructuralCandidateContext(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     scope: Literal["top_level_window"]
-    matched_fields: list[
-        Literal["automation_id", "control_type", "name"]
-    ] = Field(min_length=1)
+    matched_fields: list[Literal["automation_id", "control_type", "name"]] = Field(min_length=1)
 
 
 class StructuralObservation(BaseModel):
     """UI structure retained beside one native action event."""
 
     model_config = ConfigDict(extra="forbid")
 
-    schema_version: Literal[
-        "openadapt.capture.structural-observation/v1"
-    ] = STRUCTURAL_OBSERVATION_SCHEMA_VERSION
+    schema_version: Literal["openadapt.capture.structural-observation/v1"] = (
+        STRUCTURAL_OBSERVATION_SCHEMA_VERSION
+    )
     provider: _ProviderIdentifier = Field(
         description=(
-            "Accessibility-provider identifier. Capture currently emits "
-            "windows_uia; other providers remain opt-in observer extensions."
+            "Accessibility-provider identifier. Native Capture providers are "
+            "windows_uia, macos_ax, and linux_atspi."
         ),
     )
     event_timestamp: float
@@ -167,17 +174,32 @@ def create_structural_observer(
     if not enabled:
         return None
     resolved_platform = platform_name or sys.platform
-    if resolved_platform != "win32":
-        return None
-
     try:
-        from openadapt_capture.structural_observer.windows import (
-            WindowsUIAStructuralObserver,
-        )
-
-        return WindowsUIAStructuralObserver()
+        if resolved_platform == "win32":
+            from openadapt_capture.structural_observer.windows import (
+                WindowsUIAStructuralObserver,
+            )
+
+            return WindowsUIAStructuralObserver()
+        if resolved_platform == "darwin":
+            from openadapt_capture.structural_observer.macos import (
+                MacOSAXStructuralObserver,
+            )
+
+            return MacOSAXStructuralObserver()
+        if resolved_platform.startswith("linux"):
+            from openadapt_capture.structural_observer.linux import (
+                LinuxATSpiStructuralObserver,
+            )
+
+            return LinuxATSpiStructuralObserver()
+        return None
     except Exception as exc:
-        _logger.warning("Windows UIA structural observation is unavailable: %s", exc)
+        _logger.warning(
+            "%s structural observation is unavailable: %s",
+            resolved_platform,
+            exc,
+        )
         return None