using System.Windows.Automation;
using System.Runtime.InteropServices;
namespace WindowsBridge;
///
/// UI Automation wrapper — equivalent to macOS AccessibilityBridge.swift.
/// Uses the Windows UI Automation framework to inspect and interact with UI elements.
///
class UIAutomationBridge
{
///
/// Get the full UI element tree for a process.
///
public Dictionary GetElementTree(int pid, int maxDepth)
{
var rootElement = GetRootElementForProcess(pid);
return BuildTree(rootElement, 0, maxDepth, new List());
}
///
/// Find an element by role, title, value, or identifier.
///
public Dictionary FindElement(int pid, string? role, string? title,
string? value, string? identifier, bool exact)
{
var rootElement = GetRootElementForProcess(pid);
// Build conditions
var conditions = new List();
if (!string.IsNullOrEmpty(role))
{
var controlType = MapRoleToControlType(role!);
if (controlType != null)
conditions.Add(new PropertyCondition(AutomationElement.ControlTypeProperty, controlType));
}
if (!string.IsNullOrEmpty(title))
{
if (exact)
conditions.Add(new PropertyCondition(AutomationElement.NameProperty, title));
// For non-exact, we'll filter after search
}
if (!string.IsNullOrEmpty(identifier))
{
conditions.Add(new PropertyCondition(AutomationElement.AutomationIdProperty, identifier));
}
Condition searchCondition;
if (conditions.Count == 0)
searchCondition = Condition.TrueCondition;
else if (conditions.Count == 1)
searchCondition = conditions[0];
else
searchCondition = new AndCondition(conditions.ToArray());
AutomationElement? found;
if (!string.IsNullOrEmpty(title) && !exact)
{
// For partial match, walk the tree manually
found = FindElementByPartialName(rootElement, title!, role, 10);
}
else
{
found = rootElement.FindFirst(TreeScope.Descendants, searchCondition);
}
if (found == null)
throw new BridgeException($"Element not found: role={role}, title={title}, value={value}");
// Build element path for later reference
var elementPath = GetElementPath(rootElement, found);
var result = new Dictionary
{
["role"] = MapControlTypeToRole(found.Current.ControlType),
["title"] = found.Current.Name,
["elementPath"] = elementPath,
};
try
{
var bounds = found.Current.BoundingRectangle;
if (!bounds.IsEmpty)
{
result["bounds"] = new Dictionary
{
["x"] = bounds.X,
["y"] = bounds.Y,
["width"] = bounds.Width,
["height"] = bounds.Height,
};
}
}
catch { }
// Try to get value
try
{
if (found.TryGetCurrentPattern(ValuePattern.Pattern, out object? pattern))
{
result["value"] = ((ValuePattern)pattern).Current.Value;
}
}
catch { }
return result;
}
///
/// Perform an action on an element at the given path.
/// Maps macOS AX actions to Windows UIA patterns.
///
public Dictionary PerformAction(int pid, int[] elementPath, string action)
{
var rootElement = GetRootElementForProcess(pid);
var element = NavigateToElement(rootElement, elementPath);
switch (action)
{
case "AXPress":
case "press":
case "click":
if (element.TryGetCurrentPattern(InvokePattern.Pattern, out object? invokePattern))
{
((InvokePattern)invokePattern).Invoke();
}
else if (element.TryGetCurrentPattern(TogglePattern.Pattern, out object? togglePattern))
{
((TogglePattern)togglePattern).Toggle();
}
else if (element.TryGetCurrentPattern(SelectionItemPattern.Pattern, out object? selPattern))
{
((SelectionItemPattern)selPattern).Select();
}
else if (element.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out object? ecPattern))
{
var p = (ExpandCollapsePattern)ecPattern;
if (p.Current.ExpandCollapseState == ExpandCollapseState.Collapsed)
p.Expand();
else
p.Collapse();
}
else
{
// Fallback: click at element center
var bounds = element.Current.BoundingRectangle;
if (!bounds.IsEmpty)
{
var x = bounds.X + bounds.Width / 2;
var y = bounds.Y + bounds.Height / 2;
new InputBridge().MouseClick(x, y, "left", 1);
}
else
{
throw new BridgeException($"Element does not support any click pattern and has no bounds");
}
}
break;
case "AXShowMenu":
case "showMenu":
if (element.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out object? expandPattern))
{
((ExpandCollapsePattern)expandPattern).Expand();
}
break;
case "AXScrollToVisible":
case "scrollToVisible":
if (element.TryGetCurrentPattern(ScrollItemPattern.Pattern, out object? scrollPattern))
{
((ScrollItemPattern)scrollPattern).ScrollIntoView();
}
break;
default:
throw new BridgeException($"Unsupported action: {action}");
}
return new Dictionary { ["ok"] = true };
}
///
/// Set value of a text field or similar element.
///
public Dictionary SetElementValue(int pid, int[] elementPath, string value)
{
var rootElement = GetRootElementForProcess(pid);
var element = NavigateToElement(rootElement, elementPath);
if (element.TryGetCurrentPattern(ValuePattern.Pattern, out object? pattern))
{
((ValuePattern)pattern).SetValue(value);
}
else
{
// Fallback: focus and type
try { element.SetFocus(); } catch { }
System.Threading.Thread.Sleep(50);
// Select all and type
new InputBridge().KeyCombo(new[] { "ctrl", "a" });
System.Threading.Thread.Sleep(50);
new InputBridge().TypeText(value);
}
return new Dictionary { ["ok"] = true };
}
///
/// Get value of an element.
///
public Dictionary GetElementValue(int pid, int[] elementPath)
{
var rootElement = GetRootElementForProcess(pid);
var element = NavigateToElement(rootElement, elementPath);
string? val = null;
if (element.TryGetCurrentPattern(ValuePattern.Pattern, out object? pattern))
{
val = ((ValuePattern)pattern).Current.Value;
}
else
{
val = element.Current.Name;
}
return new Dictionary { ["value"] = val };
}
///
/// Click a menu item by path (e.g., ["File", "New"]).
///
public Dictionary MenuClick(int pid, string[] menuPath)
{
var rootElement = GetRootElementForProcess(pid);
// Find the menu bar
var menuBar = rootElement.FindFirst(TreeScope.Children,
new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.MenuBar));
if (menuBar == null)
{
// Try looking in the window's children
var window = rootElement.FindFirst(TreeScope.Children,
new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Window));
if (window != null)
{
menuBar = window.FindFirst(TreeScope.Children,
new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.MenuBar));
}
}
if (menuBar == null)
throw new BridgeException("Menu bar not found");
AutomationElement current = menuBar;
for (int i = 0; i < menuPath.Length; i++)
{
var menuName = menuPath[i];
// Find the menu item by name
var menuItem = current.FindFirst(TreeScope.Children,
new PropertyCondition(AutomationElement.NameProperty, menuName));
if (menuItem == null)
{
// Try partial match
var children = current.FindAll(TreeScope.Children, Condition.TrueCondition);
foreach (AutomationElement child in children)
{
if (child.Current.Name.Contains(menuName, StringComparison.OrdinalIgnoreCase))
{
menuItem = child;
break;
}
}
}
if (menuItem == null)
throw new BridgeException($"Menu item not found: {menuName}");
if (i < menuPath.Length - 1)
{
// Expand submenu
if (menuItem.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out object? ecPattern))
{
((ExpandCollapsePattern)ecPattern).Expand();
System.Threading.Thread.Sleep(100);
}
else if (menuItem.TryGetCurrentPattern(InvokePattern.Pattern, out object? invPattern))
{
((InvokePattern)invPattern).Invoke();
System.Threading.Thread.Sleep(100);
}
// After expanding, the submenu items should be children or in a popup
current = menuItem;
}
else
{
// Click the final menu item
if (menuItem.TryGetCurrentPattern(InvokePattern.Pattern, out object? invPattern))
{
((InvokePattern)invPattern).Invoke();
}
else if (menuItem.TryGetCurrentPattern(ExpandCollapsePattern.Pattern, out object? ecPattern))
{
((ExpandCollapsePattern)ecPattern).Expand();
}
}
}
return new Dictionary { ["ok"] = true };
}
// ── Helpers ──
private AutomationElement GetRootElementForProcess(int pid)
{
var root = AutomationElement.RootElement;
var condition = new PropertyCondition(AutomationElement.ProcessIdProperty, pid);
var element = root.FindFirst(TreeScope.Children, condition);
if (element == null)
{
// Try finding any window with this PID
var allWindows = root.FindAll(TreeScope.Children, Condition.TrueCondition);
foreach (AutomationElement win in allWindows)
{
try
{
if (win.Current.ProcessId == pid)
{
element = win;
break;
}
}
catch { }
}
}
if (element == null)
throw new BridgeException($"No window found for PID {pid}");
return element;
}
private Dictionary BuildTree(AutomationElement element, int depth, int maxDepth, List path)
{
var node = new Dictionary
{
["role"] = MapControlTypeToRole(element.Current.ControlType),
};
var name = element.Current.Name;
if (!string.IsNullOrEmpty(name))
node["title"] = name;
// Get value if available
try
{
if (element.TryGetCurrentPattern(ValuePattern.Pattern, out object? pattern))
{
var val = ((ValuePattern)pattern).Current.Value;
if (!string.IsNullOrEmpty(val))
node["value"] = val;
}
}
catch { }
// Get bounds
try
{
var bounds = element.Current.BoundingRectangle;
if (!bounds.IsEmpty)
{
node["bounds"] = new Dictionary
{
["x"] = bounds.X,
["y"] = bounds.Y,
["width"] = bounds.Width,
["height"] = bounds.Height,
};
}
}
catch { }
node["path"] = path.ToArray();
// Recurse into children
if (depth < maxDepth)
{
try
{
var children = element.FindAll(TreeScope.Children, Condition.TrueCondition);
if (children.Count > 0)
{
var childNodes = new List>();
for (int i = 0; i < children.Count && i < 100; i++) // Cap at 100 children
{
var childPath = new List(path) { i };
try
{
childNodes.Add(BuildTree(children[i], depth + 1, maxDepth, childPath));
}
catch
{
// Skip inaccessible children
}
}
if (childNodes.Count > 0)
node["children"] = childNodes;
}
}
catch { }
}
return node;
}
private AutomationElement NavigateToElement(AutomationElement root, int[] path)
{
var current = root;
foreach (var index in path)
{
var children = current.FindAll(TreeScope.Children, Condition.TrueCondition);
if (index >= children.Count)
throw new BridgeException($"Element path index {index} out of range (count={children.Count})");
current = children[index];
}
return current;
}
private int[] GetElementPath(AutomationElement root, AutomationElement target)
{
// BFS to find the path from root to target
var queue = new Queue<(AutomationElement element, List path)>();
queue.Enqueue((root, new List()));
while (queue.Count > 0)
{
var (current, path) = queue.Dequeue();
if (Automation.Compare(current, target))
return path.ToArray();
try
{
var children = current.FindAll(TreeScope.Children, Condition.TrueCondition);
for (int i = 0; i < children.Count && i < 100; i++)
{
var childPath = new List(path) { i };
queue.Enqueue((children[i], childPath));
}
}
catch { }
}
// Fallback: return empty path
return Array.Empty();
}
private AutomationElement? FindElementByPartialName(AutomationElement root, string partialName,
string? role, int maxDepth)
{
if (maxDepth <= 0) return null;
try
{
var name = root.Current.Name;
if (!string.IsNullOrEmpty(name) &&
name.Contains(partialName, StringComparison.OrdinalIgnoreCase))
{
if (role == null || MapControlTypeToRole(root.Current.ControlType)
.Equals(role, StringComparison.OrdinalIgnoreCase))
{
return root;
}
}
}
catch { }
try
{
var children = root.FindAll(TreeScope.Children, Condition.TrueCondition);
foreach (AutomationElement child in children)
{
var found = FindElementByPartialName(child, partialName, role, maxDepth - 1);
if (found != null) return found;
}
}
catch { }
return null;
}
// Map macOS AX roles to Windows UIA ControlTypes
private static ControlType? MapRoleToControlType(string role)
{
return role.ToLowerInvariant() switch
{
"button" or "axbutton" => ControlType.Button,
"checkbox" or "axcheckbox" => ControlType.CheckBox,
"combobox" or "axcombobox" => ControlType.ComboBox,
"textfield" or "axtextfield" or "textarea" or "axtextarea" => ControlType.Edit,
"group" or "axgroup" => ControlType.Group,
"image" or "aximage" => ControlType.Image,
"link" or "axlink" => ControlType.Hyperlink,
"list" or "axlist" => ControlType.List,
"menu" or "axmenu" => ControlType.Menu,
"menuitem" or "axmenuitem" => ControlType.MenuItem,
"menubar" or "axmenubar" => ControlType.MenuBar,
"radiobutton" or "axradiobutton" => ControlType.RadioButton,
"scrollbar" or "axscrollbar" => ControlType.ScrollBar,
"slider" or "axslider" => ControlType.Slider,
"statictext" or "axstatictext" => ControlType.Text,
"tab" or "axtab" or "tabgroup" or "axtabgroup" => ControlType.Tab,
"table" or "axtable" => ControlType.Table,
"toolbar" or "axtoolbar" => ControlType.ToolBar,
"tree" or "axtree" or "outline" or "axoutline" => ControlType.Tree,
"window" or "axwindow" => ControlType.Window,
_ => null,
};
}
// Map Windows UIA ControlTypes to macOS-style role strings
private static string MapControlTypeToRole(ControlType ct)
{
if (ct == ControlType.Button) return "AXButton";
if (ct == ControlType.CheckBox) return "AXCheckBox";
if (ct == ControlType.ComboBox) return "AXComboBox";
if (ct == ControlType.Edit) return "AXTextField";
if (ct == ControlType.Group) return "AXGroup";
if (ct == ControlType.Image) return "AXImage";
if (ct == ControlType.Hyperlink) return "AXLink";
if (ct == ControlType.List) return "AXList";
if (ct == ControlType.ListItem) return "AXCell";
if (ct == ControlType.Menu) return "AXMenu";
if (ct == ControlType.MenuItem) return "AXMenuItem";
if (ct == ControlType.MenuBar) return "AXMenuBar";
if (ct == ControlType.Pane) return "AXGroup";
if (ct == ControlType.RadioButton) return "AXRadioButton";
if (ct == ControlType.ScrollBar) return "AXScrollBar";
if (ct == ControlType.Slider) return "AXSlider";
if (ct == ControlType.StatusBar) return "AXStaticText";
if (ct == ControlType.Tab) return "AXTabGroup";
if (ct == ControlType.TabItem) return "AXTab";
if (ct == ControlType.Table) return "AXTable";
if (ct == ControlType.Text) return "AXStaticText";
if (ct == ControlType.ToolBar) return "AXToolbar";
if (ct == ControlType.ToolTip) return "AXStaticText";
if (ct == ControlType.Tree) return "AXOutline";
if (ct == ControlType.TreeItem) return "AXRow";
if (ct == ControlType.Window) return "AXWindow";
if (ct == ControlType.Document) return "AXWebArea";
if (ct == ControlType.Header) return "AXGroup";
if (ct == ControlType.DataGrid) return "AXTable";
if (ct == ControlType.DataItem) return "AXCell";
if (ct == ControlType.SplitButton) return "AXButton";
if (ct == ControlType.Spinner) return "AXIncrementor";
if (ct == ControlType.Thumb) return "AXHandle";
if (ct == ControlType.TitleBar) return "AXStaticText";
if (ct == ControlType.Custom) return "AXGroup";
return "AXGroup"; // Default fallback
}
}