From 878f5ce741e9a6c79f9345d3ecf73112cffa981a Mon Sep 17 00:00:00 2001 From: Jared Goodwin Date: Thu, 29 Oct 2026 14:01:31 -0700 Subject: [PATCH] Add UIA action-click tools (list_elements, perform_action, get_element_state) Windows-only tools that activate UI Automation control patterns directly, so agents can click, toggle, and expand controls without moving the pointer. Hand-written COM interop, session-scoped element ids with explicit staleness, MTA execution behind a provider timeout. Pixel click stays the documented fallback for custom-drawn surfaces. --- .../Windows/Uia/UiaElementCache.cs | 178 ++++++++ .../ComputerUse/Windows/Uia/UiaInterop.cs | 173 ++++++++ .../Windows/Uia/WindowsUiaService.cs | 389 ++++++++++++++++++ .../ComputerUse/Windows/WindowsUiaTools.cs | 95 +++++ Bitbound.ComputerUseDotnet/Program.cs | 2 + README.md | 5 +- .../WindowsUiaServiceTests.cs | 176 ++++++++ skills/computer-use-dotnet/SKILL.md | 5 + 8 files changed, 1022 insertions(+), 1 deletion(-) create mode 100644 Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/UiaElementCache.cs create mode 100644 Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/UiaInterop.cs create mode 100644 Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/WindowsUiaService.cs create mode 100644 Bitbound.ComputerUseDotnet/ComputerUse/Windows/WindowsUiaTools.cs create mode 100644 Tests/Bitbound.ComputerUseDotnet.Tests/WindowsUiaServiceTests.cs diff --git a/Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/UiaElementCache.cs b/Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/UiaElementCache.cs new file mode 100644 index 0000000..22394d7 --- /dev/null +++ b/Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/UiaElementCache.cs @@ -0,0 +1,178 @@ +using System.Globalization; +using System.Text; + +namespace Bitbound.ComputerUseDotnet.ComputerUse.Windows; + +/// Metadata for one UIA element captured during a list_elements walk. +internal sealed record UiaElementSnapshot( + string Id, + string ControlType, + string Name, + string AutomationId, + int X, + int Y, + int Width, + int Height, + IReadOnlyList Patterns); + +/// +/// Session-scoped handle table mapping element ids (e1, e2, ...) to cached UIA element objects. +/// Ids are never reused. Re-listing a window replaces that window's entries, so ids from an older +/// listing fail loudly instead of acting on a stale element. +/// +internal sealed class UiaElementCache +{ + private readonly Dictionary _elements = []; + + private int _nextId; + + public int Count => _elements.Count; + + public string Add(nint windowHandle, object element, UiaElementSnapshot snapshotWithoutId) + { + var id = $"e{Interlocked.Increment(ref _nextId)}"; + var snapshot = snapshotWithoutId with { Id = id }; + _elements[id] = new CachedElement(windowHandle, element, snapshot); + return id; + } + + /// Drops every entry belonging to a window so its old ids stop resolving. + public int RemoveWindow(nint windowHandle) + { + var stale = _elements.Where(pair => pair.Value.WindowHandle == windowHandle).Select(pair => pair.Key).ToList(); + + foreach (var id in stale) + { + _elements.Remove(id); + } + + return stale.Count; + } + + public bool TryGet(string id, out CachedElement cached) => _elements.TryGetValue(id, out cached!); + + internal sealed record CachedElement(nint WindowHandle, object Element, UiaElementSnapshot Snapshot); +} + +internal enum UiaAction +{ + Invoke, + Toggle, + Expand, + Collapse, + SetValue, + Select, + ScrollIntoView, +} + +internal static class UiaActions +{ + private static readonly Dictionary _byName = new(StringComparer.OrdinalIgnoreCase) + { + ["invoke"] = UiaAction.Invoke, + ["toggle"] = UiaAction.Toggle, + ["expand"] = UiaAction.Expand, + ["collapse"] = UiaAction.Collapse, + ["set_value"] = UiaAction.SetValue, + ["select"] = UiaAction.Select, + ["scroll_into_view"] = UiaAction.ScrollIntoView, + }; + + public static IReadOnlyList Names => _byName.Keys.ToList(); + + /// Pattern name the action needs, or null when the action needs no pattern. + public static string? RequiredPattern(UiaAction action) => action switch + { + UiaAction.Invoke => "invoke", + UiaAction.Toggle => "toggle", + UiaAction.Expand or UiaAction.Collapse => "expand_collapse", + UiaAction.SetValue => "value", + UiaAction.Select => "selection_item", + UiaAction.ScrollIntoView => "scroll_item", + _ => null, + }; + + public static bool TryParse(string action, out UiaAction parsed) => _byName.TryGetValue(action.Trim(), out parsed); + + /// Validates the value parameter against the action, throwing a tool-facing message. + public static void ValidateValue(UiaAction action, string? value) + { + if (action == UiaAction.SetValue && string.IsNullOrEmpty(value)) + { + throw new InvalidOperationException("set_value requires the value parameter."); + } + + if (action != UiaAction.SetValue && !string.IsNullOrEmpty(value)) + { + throw new InvalidOperationException($"The value parameter is only used by set_value, not '{ action.ToString().ToLowerInvariant()}'."); + } + } +} + +internal static class UiaFormat +{ + public static string Double(double value) => value.ToString("0.###", CultureInfo.InvariantCulture); + + public static string ElementLine(UiaElementSnapshot element) => + $"[{element.Id}] {element.ControlType} \"{element.Name}\"" + + (string.IsNullOrEmpty(element.AutomationId) ? string.Empty : $" automationId={element.AutomationId}") + + $" rect=({element.X}, {element.Y}) {element.Width}x{element.Height}" + + (element.Patterns.Count == 0 ? " patterns=none" : $" patterns={string.Join(",", element.Patterns)}"); + + public static string ListHeader(string windowDescription, int shown, bool truncated, int depthLimitReached) => + $"{windowDescription}: {shown} element(s) listed" + + (truncated ? " (walk stopped at the element limit; narrow the filters or raise max_elements)." : ".") + + (depthLimitReached > 0 ? $" {depthLimitReached} element(s) were at or below max_depth and their children were skipped." : string.Empty) + + " Use an id with perform_action or get_element_state. Ids expire when the window is re-listed."; + + public static string State( + UiaElementSnapshot element, + string? enabled, + string? offscreen, + string? value, + string? toggle, + string? rangeValue, + string? expandCollapse, + string? selected) + { + var builder = new StringBuilder(); + builder.AppendLine(ElementLine(element)); + + if (enabled is not null) + { + builder.AppendLine($"Enabled: {enabled}"); + } + + if (offscreen is not null) + { + builder.AppendLine($"Offscreen: {offscreen}"); + } + + if (value is not null) + { + builder.AppendLine($"Value: {value}"); + } + + if (rangeValue is not null) + { + builder.AppendLine($"RangeValue: {rangeValue}"); + } + + if (toggle is not null) + { + builder.AppendLine($"Toggle: {toggle}"); + } + + if (expandCollapse is not null) + { + builder.AppendLine($"ExpandCollapse: {expandCollapse}"); + } + + if (selected is not null) + { + builder.AppendLine($"Selected: {selected}"); + } + + return builder.ToString().TrimEnd(); + } +} diff --git a/Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/UiaInterop.cs b/Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/UiaInterop.cs new file mode 100644 index 0000000..14f35be --- /dev/null +++ b/Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/UiaInterop.cs @@ -0,0 +1,173 @@ +using System.Runtime.InteropServices; + +// COM interop members must stay in UIAutomationClient.h vtable order, which outranks member ordering. +#pragma warning disable BB0001 + +namespace Bitbound.ComputerUseDotnet.ComputerUse.Windows; + +/// +/// Hand-written UI Automation client interop for UIAutomationCore. Interface member order is the +/// vtable order from UIAutomationClient.h and must not be reordered. Slots that are never called are +/// padded with placeholder methods. Trailing slots beyond the last called method are omitted, which +/// is safe because the CLR binds by declaration index and undeclared slots are never invoked. +/// +internal static class Uia +{ + public static readonly Guid AutomationClsid = new("FF48DBA4-60EF-4201-AA87-54103EEF594E"); + + public const int PropertyBoundingRectangle = 30001; + public const int PropertyControlType = 30003; + public const int PropertyLocalizedControlType = 30004; + public const int PropertyName = 30005; + public const int PropertyIsEnabled = 30010; + public const int PropertyAutomationId = 30011; + public const int PropertyIsOffscreen = 30022; + + public const int PatternInvoke = 10000; + public const int PatternToggle = 10015; + public const int PatternExpandCollapse = 10003; + public const int PatternValue = 10002; + public const int PatternRangeValue = 10004; + public const int PatternSelectionItem = 10010; + public const int PatternScrollItem = 10017; +} + +[ComImport, Guid("30CBE57D-D9D0-452A-AB13-7AC5AC4825EE"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomation +{ + void _CompareElements(); + void _CompareRuntimeIds(); + void _GetRootElement(); + + void ElementFromHandle(nint hwnd, [MarshalAs(UnmanagedType.Interface)] out IUIAutomationElement element); + + void _ElementFromPoint(); + void _GetFocusedElement(); + void _GetRootElementBuildCache(); + void _ElementFromHandleBuildCache(); + void _ElementFromPointBuildCache(); + void _GetFocusedElementBuildCache(); + void _CreateTreeWalker(); + + void get_ControlViewWalker([MarshalAs(UnmanagedType.Interface)] out IUIAutomationTreeWalker walker); +} + +[ComImport, Guid("4042C624-389C-4AFC-A630-9DF854A541FC"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomationTreeWalker +{ + void _GetParentElement(); + + void GetFirstChildElement( + [MarshalAs(UnmanagedType.Interface)] IUIAutomationElement element, + [MarshalAs(UnmanagedType.Interface)] out IUIAutomationElement? first); + + void _GetLastChildElement(); + + void GetNextSiblingElement( + [MarshalAs(UnmanagedType.Interface)] IUIAutomationElement element, + [MarshalAs(UnmanagedType.Interface)] out IUIAutomationElement? next); +} + +[ComImport, Guid("D22108AA-8AC5-49A5-837B-37BBB3D7591E"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomationElement +{ + void _SetFocus(); + void _GetRuntimeId(); + void _FindFirst(); + void _FindAll(); + void _FindFirstBuildCache(); + void _FindAllBuildCache(); + void _BuildUpdatedCache(); + + void GetCurrentPropertyValue( + int propertyId, + [MarshalAs(UnmanagedType.Struct)] out object? value); + + void _GetCurrentPropertyValueEx(); + void _GetCachedPropertyValue(); + void _GetCachedPropertyValueEx(); + void _GetCurrentPatternAs(); + void _GetCachedPatternAs(); + + void GetCurrentPattern( + int patternId, + [MarshalAs(UnmanagedType.IUnknown)] out object? patternObject); +} + +[ComImport, Guid("FB377FBE-8EA6-46D5-9C73-6499642D3059"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomationInvokePattern +{ + void Invoke(); +} + +[ComImport, Guid("94CF8058-9B8D-4AB9-8BFD-4CD0A33C8C70"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomationTogglePattern +{ + void Toggle(); + + UiaToggleState CurrentToggleState { get; } +} + +[ComImport, Guid("619BE086-1F4E-4EE4-BAFA-210128738730"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomationExpandCollapsePattern +{ + void Expand(); + void Collapse(); + + UiaExpandCollapseState CurrentExpandCollapseState { get; } +} + +[ComImport, Guid("A94CD8B1-0844-4CD6-9D2D-640537AB39E9"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomationValuePattern +{ + void SetValue([MarshalAs(UnmanagedType.BStr)] string value); + + string CurrentValue { [return: MarshalAs(UnmanagedType.BStr)] get; } + + int CurrentIsReadOnly { get; } +} + +[ComImport, Guid("59213F4F-7346-49E5-B120-80555987A148"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomationRangeValuePattern +{ + void SetValue(double value); + + double CurrentValue { get; } + + int CurrentIsReadOnly { get; } + + double CurrentMaximum { get; } + + double CurrentMinimum { get; } +} + +[ComImport, Guid("A8EFA66A-0FDA-421A-9194-38021F3578EA"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomationSelectionItemPattern +{ + void Select(); + void _AddToSelection(); + void _RemoveFromSelection(); + + int CurrentIsSelected { get; } +} + +[ComImport, Guid("B488300F-D015-4F19-9C29-BB595E3645EF"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] +internal interface IUIAutomationScrollItemPattern +{ + void ScrollIntoView(); +} + +internal enum UiaToggleState +{ + Off = 0, + On = 1, + Indeterminate = 2, +} + +internal enum UiaExpandCollapseState +{ + Collapsed = 0, + Expanded = 1, + PartiallyExpanded = 2, + LeafNode = 3, +} diff --git a/Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/WindowsUiaService.cs b/Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/WindowsUiaService.cs new file mode 100644 index 0000000..2dbd6ba --- /dev/null +++ b/Bitbound.ComputerUseDotnet/ComputerUse/Windows/Uia/WindowsUiaService.cs @@ -0,0 +1,389 @@ +using System.Runtime.InteropServices; +using System.Runtime.Versioning; + +namespace Bitbound.ComputerUseDotnet.ComputerUse.Windows; + +/// +/// Drives UI Automation control patterns (Invoke, Toggle, ExpandCollapse, Value, SelectionItem, +/// ScrollItem) against cached elements so agents can act on controls without moving the pointer. +/// Pattern activation is app-emulated, not a real mouse click; custom-drawn surfaces ignore it. +/// All UIA calls run on thread-pool threads (MTA, as UIA requires) behind an overall timeout. +/// A timed-out call is abandoned, not aborted; its COM worker thread stays blocked until the +/// provider returns, which is acceptable for a first iteration on hung-provider protection. +/// +[SupportedOSPlatform("windows")] +public sealed class WindowsUiaService +{ + private static readonly (string Name, int PatternId)[] SupportedPatterns = + [ + ("invoke", Uia.PatternInvoke), + ("toggle", Uia.PatternToggle), + ("expand_collapse", Uia.PatternExpandCollapse), + ("value", Uia.PatternValue), + ("range_value", Uia.PatternRangeValue), + ("selection_item", Uia.PatternSelectionItem), + ("scroll_item", Uia.PatternScrollItem), + ]; + private static readonly TimeSpan UiaTimeout = TimeSpan.FromSeconds(10); + + private readonly UiaElementCache _cache = new(); + private readonly object _sync = new(); + private readonly WindowsWindowService _windows; + + private IUIAutomation? _uia; + + public WindowsUiaService(WindowsWindowService windows) + { + _windows = windows; + } + + public async Task GetElementState(string elementId) + { + var cached = Lookup(elementId); + + return await Run(() => + { + var element = AsElement(cached); + + // One property read first; if the element is dead every read would throw and this gives the + // re-run-list_elements message once instead of silently omitting every line. + element.GetCurrentPropertyValue(Uia.PropertyName, out _); + + var enabled = TryGetBoolProperty(element, Uia.PropertyIsEnabled); + var offscreen = TryGetBoolProperty(element, Uia.PropertyIsOffscreen); + + string? value = null; + if (TryGetPattern(element, "value") is IUIAutomationValuePattern valuePattern) + { + value = $"\"{valuePattern.CurrentValue}\" readonly={valuePattern.CurrentIsReadOnly != 0}"; + } + + string? rangeValue = null; + if (TryGetPattern(element, "range_value") is IUIAutomationRangeValuePattern rangePattern) + { + rangeValue = $"{UiaFormat.Double(rangePattern.CurrentValue)} (min {UiaFormat.Double(rangePattern.CurrentMinimum)}, max {UiaFormat.Double(rangePattern.CurrentMaximum)}, readonly={rangePattern.CurrentIsReadOnly != 0})"; + } + + string? toggle = null; + if (TryGetPattern(element, "toggle") is IUIAutomationTogglePattern togglePattern) + { + toggle = togglePattern.CurrentToggleState.ToString(); + } + + string? expandCollapse = null; + if (TryGetPattern(element, "expand_collapse") is IUIAutomationExpandCollapsePattern expandPattern) + { + expandCollapse = expandPattern.CurrentExpandCollapseState.ToString(); + } + + string? selected = null; + if (TryGetPattern(element, "selection_item") is IUIAutomationSelectionItemPattern selectionPattern) + { + selected = selectionPattern.CurrentIsSelected != 0 ? "yes" : "no"; + } + + return UiaFormat.State(cached.Snapshot, enabled, offscreen, value, toggle, rangeValue, expandCollapse, selected); + }); + } + + public async Task PerformAction(string elementId, string action, string? value) + { + if (!UiaActions.TryParse(action, out var parsed)) + { + throw new InvalidOperationException($"Unknown action '{action}'. Valid actions: {string.Join(", ", UiaActions.Names)}."); + } + + UiaActions.ValidateValue(parsed, value); + var setValue = parsed == UiaAction.SetValue + ? value ?? throw new InvalidOperationException("set_value requires the value parameter.") + : null; + var cached = Lookup(elementId); + + return await Run(() => + { + var element = AsElement(cached); + object Require(string patternName) => + TryGetPattern(element, patternName) + ?? throw new InvalidOperationException( + $"Element {elementId} does not support the {patternName} pattern. Re-run list_elements to check its current patterns, or use click for custom-drawn controls."); + + switch (parsed) + { + case UiaAction.Invoke: + ((IUIAutomationInvokePattern)Require("invoke")).Invoke(); + break; + case UiaAction.Toggle: + ((IUIAutomationTogglePattern)Require("toggle")).Toggle(); + break; + case UiaAction.Expand: + ((IUIAutomationExpandCollapsePattern)Require("expand_collapse")).Expand(); + break; + case UiaAction.Collapse: + ((IUIAutomationExpandCollapsePattern)Require("expand_collapse")).Collapse(); + break; + case UiaAction.SetValue: + ((IUIAutomationValuePattern)Require("value")).SetValue(setValue ?? throw new InvalidOperationException("set_value requires the value parameter.")); + break; + case UiaAction.Select: + ((IUIAutomationSelectionItemPattern)Require("selection_item")).Select(); + break; + case UiaAction.ScrollIntoView: + ((IUIAutomationScrollItemPattern)Require("scroll_item")).ScrollIntoView(); + break; + } + + var description = cached.Snapshot.Name.Length > 0 ? $"\"{cached.Snapshot.Name}\"" : cached.Snapshot.ControlType; + + return $"{parsed.ToSentence()} {cached.Snapshot.ControlType} {description} ({elementId}) succeeded. The app emulated the action; hover-only or custom-drawn UI may not have reacted. Re-list or re-read state if needed."; + }); + } + + internal async Task ListElements(string window, int maxDepth, int maxElements, string? controlType, string? name) + { + return await Run(() => + { + var info = _windows.Resolve(window); + var root = GetAutomation().ElementFromHandleRoot(info.Handle); + var walker = GetAutomation().ControlViewWalker(); + + var matched = new List(); + var truncated = false; + var depthSkipped = 0; + + // The window's old ids stop resolving the moment it is re-listed. + _cache.RemoveWindow(info.Handle); + + var stack = new Stack<(IUIAutomationElement Element, int Depth)>(); + stack.Push((root, 0)); + + while (stack.Count > 0) + { + var (element, depth) = stack.Pop(); + + var snapshot = Describe(element); + + var matches = + (controlType is null || snapshot.ControlType.Contains(controlType, StringComparison.OrdinalIgnoreCase)) && + (name is null || snapshot.Name.Contains(name, StringComparison.OrdinalIgnoreCase)); + + if (matches) + { + var id = _cache.Add(info.Handle, element, snapshot); + matched.Add(snapshot with { Id = id }); + } + + if (depth >= maxDepth) + { + depthSkipped++; + continue; + } + + // Push children in reverse so the pop order is document order, depth first. + var children = new List(); + + for (var child = walker.GetFirstChild(element); child is not null; child = walker.GetNextSibling(child)) + { + children.Add(child); + } + + for (var i = children.Count - 1; i >= 0; i--) + { + stack.Push((children[i], depth + 1)); + } + + if (matched.Count >= maxElements) + { + truncated = true; + break; + } + } + + return new ListResult(info, matched, truncated, depthSkipped); + }); + } + + private static IUIAutomationElement AsElement(UiaElementCache.CachedElement cached) => + cached.Element as IUIAutomationElement + ?? throw new InvalidOperationException($"Cached element {cached.Snapshot.Id} is corrupt. Re-run list_elements."); + + private static UiaElementSnapshot Describe(IUIAutomationElement element) + { + var controlType = TryGetStringProperty(element, Uia.PropertyLocalizedControlType) ?? "custom"; + var name = TryGetStringProperty(element, Uia.PropertyName) ?? string.Empty; + var automationId = TryGetStringProperty(element, Uia.PropertyAutomationId) ?? string.Empty; + + var x = 0; + var y = 0; + var width = 0; + var height = 0; + + element.GetCurrentPropertyValue(Uia.PropertyBoundingRectangle, out var rectValue); + + if (ToDoubles(rectValue) is { Length: 4 } rect) + { + x = (int)Math.Round(rect[0]); + y = (int)Math.Round(rect[1]); + width = (int)Math.Round(rect[2]); + height = (int)Math.Round(rect[3]); + } + + var patterns = new List(); + + foreach (var (patternName, patternId) in SupportedPatterns) + { + if (TryGetPattern(element, patternId) is not null) + { + patterns.Add(patternName); + } + } + + return new UiaElementSnapshot(string.Empty, controlType, name, automationId, x, y, width, height, patterns); + } + + private static async Task Run(Func work) + { + var task = Task.Run(work); + var finished = await Task.WhenAny(task, Task.Delay(UiaTimeout)); + + if (finished != task) + { + throw new InvalidOperationException( + $"The UI Automation provider did not answer within {(int)UiaTimeout.TotalSeconds} seconds. " + + "The app may be busy or its UIA provider hung. Retry, or fall back to the pixel tools (take_screenshot + click)."); + } + + try + { + return await task; + } + catch (COMException exception) + { + throw new InvalidOperationException( + $"The UIA element is no longer valid (0x{exception.HResult:X8}). The app likely rebuilt its tree. Re-run list_elements.", + exception); + } + } + + private static double[]? ToDoubles(object? value) => value switch + { + double[] array => array, + object[] array => array.OfType().ToArray(), + _ => null, + }; + + private static string? TryGetBoolProperty(IUIAutomationElement element, int propertyId) + { + element.GetCurrentPropertyValue(propertyId, out var value); + + return value switch + { + bool flag => flag ? "yes" : "no", + int number => number != 0 ? "yes" : "no", + _ => null, + }; + } + + private static object? TryGetPattern(IUIAutomationElement element, string patternName) + { + var patternId = SupportedPatterns.First(pair => pair.Name == patternName).PatternId; + return TryGetPattern(element, patternId); + } + + private static object? TryGetPattern(IUIAutomationElement element, int patternId) + { + try + { + element.GetCurrentPattern(patternId, out var patternObject); + return patternObject; + } + catch (COMException) + { + // Providers report unsupported patterns inconsistently (null or an error). + return null; + } + } + + private static string? TryGetStringProperty(IUIAutomationElement element, int propertyId) + { + element.GetCurrentPropertyValue(propertyId, out var value); + return value as string; + } + + private IUIAutomation GetAutomation() + { + lock (_sync) + { + if (_uia is not null) + { + return _uia; + } + + var type = Type.GetTypeFromCLSID(Uia.AutomationClsid) + ?? throw new InvalidOperationException("UIAutomationCore is not registered on this machine."); + + // The server process is any-CPU and thread-pool threads are MTA, which satisfies the + // RO_INIT_MULTITHREADED apartment UIA clients require. + var instance = Activator.CreateInstance(type) + ?? throw new InvalidOperationException("Failed to create the UIAutomation COM object."); + + _uia = (IUIAutomation)instance; + return _uia; + } + } + + private UiaElementCache.CachedElement Lookup(string elementId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(elementId); + + if (!_cache.TryGet(elementId.Trim(), out var cached)) + { + throw new InvalidOperationException($"Unknown element id '{elementId}'. Ids come from list_elements and expire when the window is re-listed. Re-run list_elements."); + } + + return cached; + } + + internal sealed record ListResult(WindowInfo Window, IReadOnlyList Elements, bool Truncated, int DepthSkipped); +} + +internal static class UiaWalkerExtensions +{ + public static IUIAutomationTreeWalker ControlViewWalker(this IUIAutomation automation) + { + automation.get_ControlViewWalker(out var walker); + return walker ?? throw new InvalidOperationException("UIAutomation returned no control view walker."); + } + + public static IUIAutomationElement ElementFromHandleRoot(this IUIAutomation automation, nint hwnd) + { + automation.ElementFromHandle(hwnd, out var element); + return element ?? throw new InvalidOperationException($"ElementFromHandle returned no element for window {WindowsWindowService.FormatHandle(hwnd)}."); + } + + public static IUIAutomationElement? GetFirstChild(this IUIAutomationTreeWalker walker, IUIAutomationElement element) + { + walker.GetFirstChildElement(element, out var first); + return first; + } + + public static IUIAutomationElement? GetNextSibling(this IUIAutomationTreeWalker walker, IUIAutomationElement element) + { + walker.GetNextSiblingElement(element, out var next); + return next; + } +} + +internal static class StringActionExtensions +{ + public static string ToSentence(this UiaAction action) => action switch + { + UiaAction.Invoke => "Invoked", + UiaAction.Toggle => "Toggled", + UiaAction.Expand => "Expanded", + UiaAction.Collapse => "Collapsed", + UiaAction.SetValue => "Set value on", + UiaAction.Select => "Selected", + UiaAction.ScrollIntoView => "Scrolled into view", + _ => action.ToString(), + }; +} diff --git a/Bitbound.ComputerUseDotnet/ComputerUse/Windows/WindowsUiaTools.cs b/Bitbound.ComputerUseDotnet/ComputerUse/Windows/WindowsUiaTools.cs new file mode 100644 index 0000000..c6cd68f --- /dev/null +++ b/Bitbound.ComputerUseDotnet/ComputerUse/Windows/WindowsUiaTools.cs @@ -0,0 +1,95 @@ +using Bitbound.ComputerUseDotnet.ComputerUse.Windows; +using Microsoft.Extensions.Logging; +using System.Runtime.Versioning; + +namespace Bitbound.ComputerUseDotnet.ComputerUse; + +/// +/// Windows-only MCP tools that activate UI Automation controls through their control patterns, so +/// agents can click, toggle, and expand without moving the real pointer. Registered only when the +/// server runs on Windows. +/// +[SupportedOSPlatform("windows")] +[McpServerToolType] +public sealed class WindowsUiaTools(WindowsUiaService uia, ILogger logger) +{ + private readonly ILogger _logger = logger; + private readonly WindowsUiaService _uia = uia; + + [McpServerTool(Name = "get_element_state")] + [Description( + "Reads one UIA element by its id from list_elements (Windows only) without a screenshot: " + + "name, control type, enabled, offscreen, Value/RangeValue, Toggle state, ExpandCollapse state, and " + + "selection. Use it to verify an action or read control state. Fails if the id is unknown or stale; " + + "re-run list_elements.")] + public async Task GetElementState( + [Description("Element id from list_elements (e.g. e3).")] + string element_id) => + await ComputerUseTools.RunToolAsync("get_element_state", _logger, () => + _uia.GetElementState(element_id)); + + [McpServerTool(Name = "list_elements")] + [Description( + "Walks the UI Automation (UIA) tree of one desktop window (Windows only) and lists actionable " + + "elements with stable ids (e1, e2, ...) for use with perform_action and get_element_state. " + + "Each line gives id, control type, name, automation id, bounding rect in physical pixels, and " + + "supported patterns (invoke, toggle, expand_collapse, value, range_value, selection_item, scroll_item). " + + "Ids expire when the same window is listed again, and acting on a stale id fails; re-run this tool then. " + + "Only UIA-exposed controls appear. Canvas, game, and some Electron/Avalonia surfaces expose nothing; use " + + "capture_window + click for those. Prefer these tools over pixel clicks when elements are listed: " + + "pattern activation is exact and does not move the pointer.")] + public async Task ListElements( + [Description("Window handle (0x-prefixed hex or decimal) or a title substring, as in list_windows.")] + string window, + [Description("Maximum tree depth to walk. The window root is depth 0. Clamped to 1..100, default 25.")] + int max_depth = 25, + [Description("Maximum number of elements to return. Clamped to 1..1000, default 200.")] + int max_elements = 200, + [Description("Optional case-insensitive substring filter on the localized control type (e.g. 'button').")] + string? control_type = null, + [Description("Optional case-insensitive substring filter on the element name.")] + string? name = null) => + await ComputerUseTools.RunToolAsync("list_elements", _logger, async () => + { + var depth = Math.Clamp(max_depth, 1, 100); + var count = Math.Clamp(max_elements, 1, 1000); + + var result = await _uia.ListElements(window, depth, count, control_type, name); + + if (result.Elements.Count == 0) + { + return $"{result.Window.Title}: no UIA elements matched. The window may expose no UIA tree (custom-drawn or GPU-composited UI); use capture_window + click instead. " + + "Widen max_depth or drop the control_type/name filters if the tree is deeper than expected."; + } + + var lines = new List + { + UiaFormat.ListHeader( + $"Window {WindowsWindowService.FormatHandle(result.Window.Handle)} \"{result.Window.Title}\"", + result.Elements.Count, + result.Truncated, + result.DepthSkipped), + }; + + lines.AddRange(result.Elements.Select(UiaFormat.ElementLine)); + return string.Join(Environment.NewLine, lines); + }); + + [McpServerTool(Name = "perform_action")] + [Description( + "Activates a UIA element by its id from list_elements (Windows only) through its control pattern: " + + "invoke (button press), toggle (checkbox), expand / collapse (tree, combo, expander), set_value " + + "(Value pattern, needs the value parameter), select (SelectionItem), scroll_into_view. The real " + + "pointer is not moved and no input is synthesized. The app emulates the action, so hover states and " + + "custom-drawn controls will not react; fall back to click with desktop fractions for those. " + + "Fails if the id is unknown or stale; re-run list_elements.")] + public async Task PerformAction( + [Description("Element id from list_elements (e.g. e3).")] + string element_id, + [Description("Action to perform: invoke, toggle, expand, collapse, set_value, select, scroll_into_view.")] + string action, + [Description("Required by set_value; must be omitted for other actions.")] + string? value = null) => + await ComputerUseTools.RunToolAsync("perform_action", _logger, () => + _uia.PerformAction(element_id, action, value)); +} diff --git a/Bitbound.ComputerUseDotnet/Program.cs b/Bitbound.ComputerUseDotnet/Program.cs index 30dff82..7dff7f0 100644 --- a/Bitbound.ComputerUseDotnet/Program.cs +++ b/Bitbound.ComputerUseDotnet/Program.cs @@ -28,7 +28,9 @@ if (OperatingSystem.IsWindows()) { builder.Services.AddSingleton(); + builder.Services.AddSingleton(); mcpServer.WithTools(); + mcpServer.WithTools(); } await builder.Build().RunAsync(); diff --git a/README.md b/README.md index f7b6d3b..a88f6eb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Bitbound.ComputerUseDotnet -A cross-platform [Model Context Protocol](https://modelcontextprotocol.io) (MCP) server that lets an agent **see the screen** and **simulate input** (mouse + keyboard) on Windows, macOS, and Linux (X11 and Wayland). +A cross-platform [Model Context Protocol](https://modelcontextprotocol.io) (MCP) server that lets an agent **see the screen** and **simulate input** (mouse + keyboard) on Windows, macOS, and Linux (X11 and Wayland). On Windows it can also **act on UI Automation controls directly**, clicking and toggling without moving the pointer. ## Requirements @@ -37,6 +37,9 @@ Add the following to your GitHub Copilot config (`mcp.json`): | `list_windows` | **Windows only.** Lists top-level windows (handle, title, class, process, bounds, state). Alt-tab-eligible by default; `include_invisible` / `include_tool_windows` / `title` / `process` widen it. | | `get_window_info` | **Windows only.** Full detail for one window, by handle from `list_windows` or by title substring. | | `capture_window` | **Windows only.** Captures a single window with `PrintWindow` (renders from the window's own DC, so it works while covered) and returns PNG content plus metadata. Optional `max_side`. | +| `list_elements` | **Windows only.** Walks a window's UIA tree and lists actionable elements with ids (`e1`, `e2`, ...), control type, name, automation id, bounding rect, and supported patterns. Bounds: `max_depth` (default 25), `max_elements` (default 200), optional `control_type` / `name` filters. Ids expire when the window is re-listed. | +| `perform_action` | **Windows only.** Activates a `list_elements` id through its UIA control pattern (`invoke`, `toggle`, `expand`, `collapse`, `set_value`, `select`, `scroll_into_view`) without moving the pointer. App-emulated, so custom-drawn controls need `click` instead. | +| `get_element_state` | **Windows only.** Reads one element's state (name, enabled, Value/RangeValue, Toggle, ExpandCollapse, selection) without a screenshot. | | `check_permissions` | Reports screen-capture / input permission state without prompting. | | `request_permissions` | Asks the OS/portal for missing permissions (prompts the user where supported). | | `move_mouse` | Warps the pointer to a desktop fraction. | diff --git a/Tests/Bitbound.ComputerUseDotnet.Tests/WindowsUiaServiceTests.cs b/Tests/Bitbound.ComputerUseDotnet.Tests/WindowsUiaServiceTests.cs new file mode 100644 index 0000000..f2fc8cc --- /dev/null +++ b/Tests/Bitbound.ComputerUseDotnet.Tests/WindowsUiaServiceTests.cs @@ -0,0 +1,176 @@ +using Bitbound.ComputerUseDotnet.ComputerUse.Windows; + +namespace Bitbound.ComputerUseDotnet.Tests; + +/// Tests for the platform-independent UIA element cache, action table, and formatting. +public sealed class WindowsUiaServiceTests +{ + [Fact] + public void Add_WhenCalled_ReturnsIncrementingIds() + { + var cache = new UiaElementCache(); + + var first = cache.Add(1, new object(), Snapshot()); + var second = cache.Add(2, new object(), Snapshot()); + + Assert.Equal("e1", first); + Assert.Equal("e2", second); + Assert.Equal(2, cache.Count); + } + + [Fact] + public void Add_WhenSnapshotHasNoId_StampsTheGeneratedId() + { + var cache = new UiaElementCache(); + + var id = cache.Add(1, new object(), Snapshot()); + + Assert.True(cache.TryGet(id, out var cached)); + Assert.Equal(id, cached.Snapshot.Id); + } + + [Fact] + public void TryGet_WithUnknownId_ReturnsFalse() + { + var cache = new UiaElementCache(); + + Assert.False(cache.TryGet("e99", out _)); + } + + [Fact] + public void RemoveWindow_WhenTwoWindowsCached_DropsOnlyThatWindow() + { + var cache = new UiaElementCache(); + var kept = cache.Add(10, new object(), Snapshot()); + var dropped = cache.Add(20, new object(), Snapshot()); + + var removed = cache.RemoveWindow(20); + + Assert.Equal(1, removed); + Assert.True(cache.TryGet(kept, out _)); + Assert.False(cache.TryGet(dropped, out _)); + } + + [Fact] + public void Add_AfterRemovingWindow_ContinuesNumberingSoOldIdsNeverResolve() + { + var cache = new UiaElementCache(); + var oldId = cache.Add(10, new object(), Snapshot()); + cache.RemoveWindow(10); + + var newId = cache.Add(10, new object(), Snapshot()); + + Assert.NotEqual(oldId, newId); + Assert.False(cache.TryGet(oldId, out _)); + } + + [Theory] + [InlineData("invoke", "Invoke")] + [InlineData("TOGGLE", "Toggle")] + [InlineData(" expand ", "Expand")] + [InlineData("collapse", "Collapse")] + [InlineData("set_value", "SetValue")] + [InlineData("select", "Select")] + [InlineData("scroll_into_view", "ScrollIntoView")] + public void TryParse_WithKnownAction_Parses(string action, string expected) + { + Assert.True(UiaActions.TryParse(action, out var parsed)); + Assert.Equal(expected, parsed.ToString()); + } + + [Theory] + [InlineData("")] + [InlineData("click")] + [InlineData("double_click")] + [InlineData("expand_collapse")] + public void TryParse_WithUnknownAction_ReturnsFalse(string action) + { + Assert.False(UiaActions.TryParse(action, out _)); + } + + [Fact] + public void TryParse_WithPaddedInvoke_Parses() + { + Assert.True(UiaActions.TryParse(" invoke ", out var parsed)); + Assert.Equal(UiaAction.Invoke, parsed); + } + + [Fact] + public void ValidateValue_WhenSetValueHasNoValue_Throws() + { + Assert.Throws(() => UiaActions.ValidateValue(UiaAction.SetValue, null)); + Assert.Throws(() => UiaActions.ValidateValue(UiaAction.SetValue, string.Empty)); + } + + [Fact] + public void ValidateValue_WhenSetValueHasValue_DoesNotThrow() + { + UiaActions.ValidateValue(UiaAction.SetValue, "hello"); + } + + [Fact] + public void ValidateValue_WhenNonSetValueActionHasValue_Throws() + { + Assert.Throws(() => UiaActions.ValidateValue(UiaAction.Invoke, "hello")); + } + + [Theory] + [InlineData("invoke", "invoke")] + [InlineData("toggle", "toggle")] + [InlineData("expand", "expand_collapse")] + [InlineData("collapse", "expand_collapse")] + [InlineData("set_value", "value")] + [InlineData("select", "selection_item")] + [InlineData("scroll_into_view", "scroll_item")] + public void RequiredPattern_ReturnsExpectedPattern(string action, string expected) + { + Assert.True(UiaActions.TryParse(action, out var parsed)); + Assert.Equal(expected, UiaActions.RequiredPattern(parsed)); + } + + [Fact] + public void ElementLine_FormatsIdTypeRectAndPatterns() + { + var line = UiaFormat.ElementLine(Snapshot(id: "e3", name: "OK", automationId: "OkButton", patterns: ["invoke", "scroll_item"])); + + Assert.Equal("[e3] Button \"OK\" automationId=OkButton rect=(10, 20) 80x24 patterns=invoke,scroll_item", line); + } + + [Fact] + public void ElementLine_WhenAutomationIdEmpty_OmitsIt() + { + var line = UiaFormat.ElementLine(Snapshot(name: "Send", automationId: string.Empty, patterns: [])); + + Assert.Equal("[e1] Button \"Send\" rect=(10, 20) 80x24 patterns=none", line); + } + + [Fact] + public void ListHeader_WhenTruncated_MentionsTheElementLimit() + { + var header = UiaFormat.ListHeader("Window 0x1 \"Note\"", 200, truncated: true, depthLimitReached: 3); + + Assert.Contains("200 element(s) listed", header); + Assert.Contains("element limit", header); + Assert.Contains("max_depth", header); + } + + [Fact] + public void State_WithOnlyToggle_OmitsTheOtherSections() + { + var text = UiaFormat.State(Snapshot(id: "e2"), enabled: "yes", offscreen: null, value: null, toggle: "On", rangeValue: null, expandCollapse: null, selected: null); + + Assert.Contains("Toggle: On", text); + Assert.Contains("Enabled: yes", text); + Assert.DoesNotContain("Value:", text); + Assert.DoesNotContain("RangeValue:", text); + Assert.DoesNotContain("Selected:", text); + } + + private static UiaElementSnapshot Snapshot( + string id = "e1", + string controlType = "Button", + string name = "OK", + string automationId = "OkButton", + IReadOnlyList? patterns = null) => + new(id, controlType, name, automationId, 10, 20, 80, 24, patterns ?? ["invoke"]); +} diff --git a/skills/computer-use-dotnet/SKILL.md b/skills/computer-use-dotnet/SKILL.md index 9ce2c24..c3ce160 100644 --- a/skills/computer-use-dotnet/SKILL.md +++ b/skills/computer-use-dotnet/SKILL.md @@ -43,6 +43,9 @@ Every input result, and `get_cursor_position`, echoes the resolved fraction and | `list_windows` | none | Windows only. Lists top-level windows with handle, title, class, process, and bounds. Filters: `include_invisible`, `include_tool_windows`, `title`, `process`. | | `get_window_info` | `window` | Windows only. One window's detail. `window` is a handle from `list_windows` (e.g. `0x1A2B`) or a title substring. | | `capture_window` | `window`, `max_side=0` | Windows only. PrintWindow capture of a single window (works while covered). May render GPU-composited content blank; fall back to `take_screenshot`. | +| `list_elements` | `window`, `max_depth=25`, `max_elements=200`, `control_type`, `name` | Windows only. Lists UIA elements with ids (`e1`, ...), control type, name, rect, and patterns. Prefer these over pixel clicks when elements are listed. Ids expire when the window is re-listed. | +| `perform_action` | `element_id`, `action`, `value` | Windows only. UIA pattern activation without moving the pointer: `invoke`, `toggle`, `expand`, `collapse`, `set_value` (needs `value`), `select`, `scroll_into_view`. App-emulated; custom-drawn UI needs `click`. | +| `get_element_state` | `element_id` | Windows only. Reads an element's Name/Value/Toggle/RangeValue/ExpandCollapse/selection without a screenshot. | | `check_permissions` | none | Reports capture and input permission state without prompting. | | `request_permissions` | none | Triggers the OS or portal prompts for missing permissions. | | `move_mouse` | `x`, `y` | Moves the pointer without clicking. | @@ -69,6 +72,8 @@ Two habits keep this reliable. - Click a field before typing into it. `type_text` goes to whatever holds keyboard focus. - Capture again after any action that redraws the screen. The UI moves, and a stale screenshot will point you at the old location. If a step appears to do nothing, an immediate screenshot catching a half-drawn frame is the usual cause, so capture again. +On Windows, try UIA before pixels for standard controls. `list_elements` on the window first. If the target shows up with an `invoke`, `toggle`, or `value` pattern, use `perform_action`. It is exact, works while the window is covered, and leaves the pointer alone. Fall back to the see-act loop when the tree is empty (canvas, game, and many Electron surfaces expose nothing) or when an action reports success but the screen shows nothing happened, because pattern activation is app-emulated and hover-only or custom-drawn UI ignores it. + ## Platforms and permissions - Windows uses GDI capture and `SendInput`. No permissions are required.