/** * Windows UI Automation tool * * Reads and operates UI controls of a target program via System.Windows.Automation * (Managed UIAutomation). Runs a PowerShell script that loads UIAutomationClient / * UIAutomationTypes and drives the automation tree. * * Registers a single `ui_automation` tool with an `action` switch. * * Actions: * list_windows - List top-level windows (process / title / handle) * list_controls - Enumerate the descendant control tree of a window * find - Find controls matching name / automationId / controlType * click - Invoke a control (InvokePattern / default action / focus+enter) * click_coords - Simulate a mouse click at absolute screen coords (x, y) * set_value - Set the value of an editable control (ValuePattern) * get_value - Read the current value of an editable control (ValuePattern) * get_text - Read the text content of a control (TextPattern / Name) * focus - Move focus to a control * toggle - Toggle a control (TogglePattern, e.g. checkbox, switch) * set_range - Set a range value (RangeValuePattern, e.g. slider) * get_range - Read a range value (RangeValuePattern) * select - Select a control (SelectionItemPattern) * scroll - Scroll within a control (ScrollPattern) * expand / collapse - Expand or collapse a control (ExpandCollapsePattern) * window_minimize / window_maximize / window_restore / window_close / window_move - WindowPattern * * Target identification: a window can be located by process name/PID (process), * window title (window_title), or exact window handle (window_handle). */ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; import { StringEnum } from "@earendil-works/pi-ai"; import { writeFile, unlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; const IS_WINDOWS = process.platform === "win32"; const ACTION = StringEnum( [ "list_windows", "list_controls", "find", "click", "click_coords", "set_value", "get_value", "get_text", "focus", "toggle", "set_range", "get_range", "select", "scroll", "expand", "collapse", "window_minimize", "window_maximize", "window_restore", "window_close", "window_move", ] as const, { description: "Action to perform. Control actions: list_windows (list top-level windows), list_controls (enumerate control tree), find (find controls by name/controlType), click (invoke a control), click_coords (mouse click at screen x,y), set_value (set edit control value), get_value (read edit value), get_text (read a control's text), focus (focus a control), toggle (toggle checkbox/switch), set_range/get_range (slider/range), select (select an item), scroll (scroll a control), expand, collapse. Window actions: window_minimize, window_maximize, window_restore, window_close, window_move.", }, ); const BTN = StringEnum(["left", "right", "middle"] as const, { description: "Mouse button for click_coords (default left).", }); const SCROLL_H = StringEnum(["horizontal", "vertical", "both"] as const, { description: "Scroll axis for scroll action.", }); const REL_TO = StringEnum(["window", "desktop"] as const, { description: "Origin for relative click_coords when relative=true. 'window' = offset from the target window's top-left (requires a window target); 'desktop' = offset from the virtual desktop's top-left (no window target needed).", }); const Params = Type.Object({ action: ACTION, // Target window: any of these can identify it. process: Type.Optional( Type.String({ description: "Target process name (e.g. 'notepad') or PID to locate its window(s)." }), ), window_title: Type.Optional(Type.String({ description: "Target window title (substring match)." })), window_handle: Type.Optional(Type.String({ description: "Exact window handle (hex, e.g. '0x00000000')." })), // Control matching: combine these to narrow the search. name: Type.Optional(Type.String({ description: "Control name to match (substring, case-insensitive)." })), automation_id: Type.Optional(Type.String({ description: "Control automation id to match (exact)." })), control_type: Type.Optional( Type.String({ description: "Control type name to match, e.g. Button, Edit, Text, ListItem, CheckBox, ComboBox, MenuItem, TabItem.", }), ), // Values. value: Type.Optional(Type.String({ description: "Value to set for set_value, or toggle target for toggle." })), value2: Type.Optional(Type.Number({ description: "Range value for set_range/get_range; or window move y." })), x: Type.Optional(Type.Integer({ description: "X for click_coords or window_move. When relative=true this is an offset from the relative_to origin (window or desktop top-left); otherwise an absolute screen X." })), y: Type.Optional(Type.Integer({ description: "Y for click_coords or window_move. When relative=true this is an offset from the relative_to origin (window or desktop top-left); otherwise an absolute screen Y." })), relative: Type.Optional(Type.Boolean({ description: "For click_coords: if true, x/y are offsets from the relative_to origin instead of absolute screen coords. Default false = absolute screen coords." })), relative_to: Type.Optional(REL_TO), button: Type.Optional(BTN), click_count: Type.Optional(Type.Integer({ description: "Number of clicks for click_coords (default 1)." })), amount: Type.Optional( Type.Integer({ description: "Scroll amount for scroll action (positive = forward/down, negative = back/up)." }), ), scroll_axis: Type.Optional(SCROLL_H), // Misc. max_results: Type.Optional(Type.Integer({ description: "Max results to return (default 100)." })), }); function controlTypeName(t: any): string { return t?.ProgrammaticName?.replace("ControlType.", "") ?? "Unknown"; } const UI_SCRIPT = ` param([string]$Action, [string]$Process, [string]$WindowTitle, [string]$WindowHandle, [string]$Name, [string]$AutomationId, [string]$ControlType, [string]$Value, [double]$Value2, [int]$X, [int]$Y, [string]$Button, [int]$ClickCount, [int]$Amount, [string]$ScrollAxis, [int]$MaxResults, [string]$Relative, [string]$RelativeTo) $ErrorActionPreference = 'Stop' Add-Type -AssemblyName UIAutomationClient, UIAutomationTypes Add-Type -AssemblyName System.Windows.Forms Add-Type -AssemblyName System.Drawing Add-Type @" using System.Runtime.InteropServices; public static class NativeMouse { [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint flags, int dx, int dy, int data, int extra); public static void Click(int x, int y, int count, int button) { SetCursorPos(x, y); uint down, up; if (button == 2) { down = 0x0008; up = 0x0010; } // middle else if (button == 3) { down = 0x0020; up = 0x0040; } // right else { down = 0x0002; up = 0x0004; } // left for (int i = 0; i < count; i++) { mouse_event(down, 0, 0, 0, 0); mouse_event(up, 0, 0, 0, 0); } } public static void Move(int x, int y) { SetCursorPos(x, y); } } "@ function controlTypeName($t) { if ($t -and $t.ProgrammaticName) { return $t.ProgrammaticName.Replace('ControlType.', '') } return 'Unknown' } function Get-ByProcess { param([string]$name) $procs = Get-Process -Name $name -ErrorAction SilentlyContinue if (-not $procs) { throw "No process named '$name'." } $null = $procs[0].MainWindowHandle return $procs } function Locate-Window { # Returns an AutomationElement for the target window, or $null for desktop (root). if ($WindowHandle) { $h = [IntPtr]::new([Convert]::ToInt64($WindowHandle, 16)) return [System.Windows.Automation.AutomationElement]::FromHandle($h) } $root = [System.Windows.Automation.AutomationElement]::RootElement if ($Process) { $proc = Get-Process -Name $Process -ErrorAction SilentlyContinue | Select-Object -First 1 if (-not $proc) { throw "No process named '$Process'." } $cond = New-Object System.Windows.Automation.PropertyCondition( [System.Windows.Automation.AutomationElement]::ProcessIdProperty, $proc.Id) $win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond) if ($win) { return $win } throw "No window found for process '$Process' (handle=$($proc.MainWindowHandle))." } if ($WindowTitle) { $cond = New-Object System.Windows.Automation.PropertyCondition( [System.Windows.Automation.AutomationElement]::NameProperty, $WindowTitle) $win = $root.FindFirst([System.Windows.Automation.TreeScope]::Children, $cond) if ($win) { return $win } $all = $root.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition) foreach ($w in $all) { if ($w.Current.Name -and $w.Current.Name -like "*$WindowTitle*") { return $w } } throw "No window with title matching '$WindowTitle'." } return $root # desktop } function Build-Condition { $conds = @() if ($Name) { $conds += New-Object System.Windows.Automation.PropertyCondition( [System.Windows.Automation.AutomationElement]::NameProperty, $Name) } if ($AutomationId) { $conds += New-Object System.Windows.Automation.PropertyCondition( [System.Windows.Automation.AutomationElement]::AutomationIdProperty, $AutomationId) } if ($ControlType) { $ct = [System.Windows.Automation.ControlType]::$ControlType $conds += New-Object System.Windows.Automation.PropertyCondition( [System.Windows.Automation.AutomationElement]::ControlTypeProperty, $ct) } if ($conds.Count -eq 0) { return [System.Windows.Automation.Condition]::TrueCondition } if ($conds.Count -eq 1) { return $conds[0] } return New-Object System.Windows.Automation.AndCondition($conds) } function Get-Patterns($el) { # Report which common patterns a control supports. $names = @() foreach ($p in @( [System.Windows.Automation.InvokePattern]::Pattern, [System.Windows.Automation.ValuePattern]::Pattern, [System.Windows.Automation.TogglePattern]::Pattern, [System.Windows.Automation.RangeValuePattern]::Pattern, [System.Windows.Automation.SelectionItemPattern]::Pattern, [System.Windows.Automation.SelectionPattern]::Pattern, [System.Windows.Automation.ExpandCollapsePattern]::Pattern, [System.Windows.Automation.ScrollPattern]::Pattern, [System.Windows.Automation.TextPattern]::Pattern, [System.Windows.Automation.WindowPattern]::Pattern)) { $x = $null if ($el.TryGetCurrentPattern($p, [ref]$x)) { $names += $p.ProgrammaticName } } return $names } function Describe-Element($el) { $ct = controlTypeName $el.Current.ControlType $val = $null $vpp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$vpp)) { $val = $vpp.Current.Value } $text = $null $tp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.TextPattern]::Pattern, [ref]$tp)) { $text = $tp.DocumentRange.GetText(-1) } return @{ name = $el.Current.Name id = $el.Current.AutomationId type = $ct class = $el.Current.ClassName enabled = $el.Current.IsEnabled offscreen = $el.Current.IsOffscreen focusable = $el.Current.IsKeyboardFocusable bounds = @{ x = [int]$el.Current.BoundingRectangle.X; y = [int]$el.Current.BoundingRectangle.Y; w = [int]$el.Current.BoundingRectangle.Width; h = [int]$el.Current.BoundingRectangle.Height } value = $val text = if ($text) { $text.Substring(0, [Math]::Min(200, $text.Length)) } else { $null } patterns = (Get-Patterns $el) } } function Get-ConditionText($el) { $ct = controlTypeName $el.Current.ControlType $s = "" if ($el.Current.Name) { $s += "name='$($el.Current.Name)' " } if ($el.Current.AutomationId) { $s += "id='$($el.Current.AutomationId)' " } $s += "type=$ct" if ($el.Current.ClassName) { $s += " class='$($el.Current.ClassName)'" } return $s } function Find-One($winEl) { $cond = Build-Condition $el = $winEl.FindFirst([System.Windows.Automation.TreeScope]::Descendants, $cond) if (-not $el) { throw "No control matched (name='$Name' id='$AutomationId' type='$ControlType')." } return $el } $script:Results = [System.Collections.ArrayList]::new() $script:Max = if ($MaxResults -gt 0) { $MaxResults } else { 100 } try { $winEl = Locate-Window switch ($Action) { 'list_windows' { $root = [System.Windows.Automation.AutomationElement]::RootElement $wins = $root.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition) foreach ($w in $wins) { if (-not $w.Current.Name) { continue } [void]$script:Results.Add(@{ title = $w.Current.Name pid = $w.Current.ProcessId handle = '0x' + $w.Current.NativeWindowHandle.ToString('X') type = controlTypeName $w.Current.ControlType }) if ($script:Results.Count -ge $script:Max) { break } } } 'list_controls' { $cond = [System.Windows.Automation.Condition]::TrueCondition $els = $winEl.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond) foreach ($e in $els) { [void]$script:Results.Add((Describe-Element $e)) if ($script:Results.Count -ge $script:Max) { break } } } 'find' { $cond = Build-Condition $els = $winEl.FindAll([System.Windows.Automation.TreeScope]::Descendants, $cond) foreach ($e in $els) { [void]$script:Results.Add((Describe-Element $e)) if ($script:Results.Count -ge $script:Max) { break } } } 'click' { $el = Find-One $winEl $invoke = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.InvokePattern]::Pattern, [ref]$invoke)) { $invoke.Invoke() [void]$script:Results.Add(@{ ok = $true; action = 'invoke'; matched = (Get-ConditionText $el) }) } else { # Fallback: set focus and click the control's center via native mouse. $el.SetFocus() $rect = $el.Current.BoundingRectangle [NativeMouse]::Click([int]($rect.X + $rect.Width/2), [int]($rect.Y + $rect.Height/2), 1, 1) [void]$script:Results.Add(@{ ok = $true; action = 'focus+click'; matched = (Get-ConditionText $el) }) } } 'click_coords' { if ($null -eq $X -or $null -eq $Y) { throw "click_coords requires x and y." } $c = if ($ClickCount -gt 0) { $ClickCount } else { 1 } $b = if ($Button -eq 'right') { 3 } elseif ($Button -eq 'middle') { 2 } else { 1 } $absX = $X $absY = $Y $rel = ($Relative -eq 'true') if ($rel) { # Choose the origin for relative coords. 'desktop' uses the virtual desktop # top-left (all monitors); 'window' uses the target window's top-left. $relTo = if ($RelativeTo -eq 'desktop') { 'desktop' } else { 'window' } $originX = 0 $originY = 0 if ($relTo -eq 'desktop') { # Virtual desktop top-left across all monitors (System.Windows.Forms). $vs = [System.Windows.Forms.SystemInformation]::VirtualScreen $originX = $vs.Left $originY = $vs.Top } else { # Relative to the target window. If a real window was located, use its # bounding rectangle; otherwise fall back to the virtual desktop origin. $wb = $winEl.Current.BoundingRectangle if ($wb -and $wb.Width -gt 0) { $originX = [int]$wb.X $originY = [int]$wb.Y } else { $vs = [System.Windows.Forms.SystemInformation]::VirtualScreen $originX = $vs.Left $originY = $vs.Top } } $absX = $originX + $X $absY = $originY + $Y } [NativeMouse]::Click($absX, $absY, $c, $b) [void]$script:Results.Add(@{ ok = $true; action = 'click_coords'; x = $absX; y = $absY; rel = $rel; relTo = $RelativeTo; offsetX = $X; offsetY = $Y; count = $c; button = $Button }) } 'set_value' { if ($null -eq $Value) { throw "set_value requires 'value'." } $el = Find-One $winEl $vp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$vp)) { $vp.SetValue($Value) [void]$script:Results.Add(@{ ok = $true; action = 'set_value'; matched = (Get-ConditionText $el); value = $vp.Current.Value }) } else { throw "Control does not support ValuePattern." } } 'get_value' { $el = Find-One $winEl $vp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$vp)) { [void]$script:Results.Add(@{ ok = $true; value = $vp.Current.Value; matched = (Get-ConditionText $el) }) } else { [void]$script:Results.Add(@{ ok = $false; note = 'No ValuePattern'; matched = (Get-ConditionText $el); value = $el.Current.Name }) } } 'get_text' { $el = Find-One $winEl $tp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.TextPattern]::Pattern, [ref]$tp)) { $txt = $tp.DocumentRange.GetText(-1) [void]$script:Results.Add(@{ ok = $true; text = $txt; matched = (Get-ConditionText $el) }) } else { [void]$script:Results.Add(@{ ok = $true; text = $el.Current.Name; note = 'Fallback to Name (no TextPattern)'; matched = (Get-ConditionText $el) }) } } 'focus' { $el = Find-One $winEl $el.SetFocus() [void]$script:Results.Add(@{ ok = $true; action = 'focus'; matched = (Get-ConditionText $el) }) } 'toggle' { $el = Find-One $winEl $tp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern, [ref]$tp)) { $before = $tp.Current.ToggleState $tp.Toggle() [void]$script:Results.Add(@{ ok = $true; action = 'toggle'; before = $before.ToString(); matched = (Get-ConditionText $el) }) } else { throw "Control does not support TogglePattern." } } 'set_range' { if ($null -eq $Value2) { throw "set_range requires value2." } $el = Find-One $winEl $rp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.RangeValuePattern]::Pattern, [ref]$rp)) { $rp.SetValue($Value2) [void]$script:Results.Add(@{ ok = $true; action = 'set_range'; value = $rp.Current.Value; minimum = $rp.Current.Minimum; maximum = $rp.Current.Maximum; matched = (Get-ConditionText $el) }) } else { throw "Control does not support RangeValuePattern." } } 'get_range' { $el = Find-One $winEl $rp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.RangeValuePattern]::Pattern, [ref]$rp)) { [void]$script:Results.Add(@{ ok = $true; value = $rp.Current.Value; minimum = $rp.Current.Minimum; maximum = $rp.Current.Maximum; matched = (Get-ConditionText $el) }) } else { [void]$script:Results.Add(@{ ok = $false; note = 'No RangeValuePattern'; matched = (Get-ConditionText $el) }) } } 'select' { $el = Find-One $winEl $sp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.SelectionItemPattern]::Pattern, [ref]$sp)) { $sp.Select() [void]$script:Results.Add(@{ ok = $true; action = 'select'; matched = (Get-ConditionText $el) }) } else { throw "Control does not support SelectionItemPattern." } } 'scroll' { $el = Find-One $winEl $scp = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.ScrollPattern]::Pattern, [ref]$scp)) { $a = if ($Amount -ne 0) { $Amount } else { 10 } $axis = if ($ScrollAxis) { $ScrollAxis } else { 'vertical' } $hp = [System.Windows.Automation.ScrollPattern]::NoScroll $vp = [System.Windows.Automation.ScrollPattern]::NoScroll if ($axis -eq 'horizontal' -or $axis -eq 'both') { $hp = $a } if ($axis -eq 'vertical' -or $axis -eq 'both') { $vp = $a } $scp.Scroll($hp, $vp) [void]$script:Results.Add(@{ ok = $true; action = 'scroll'; horizontal = $hp; vertical = $vp; matched = (Get-ConditionText $el) }) } else { throw "Control does not support ScrollPattern." } } 'expand' { $el = Find-One $winEl $p = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.ExpandCollapsePattern]::Pattern, [ref]$p)) { $p.Expand() [void]$script:Results.Add(@{ ok = $true; action = 'expand'; matched = (Get-ConditionText $el) }) } else { throw "Control does not support ExpandCollapsePattern." } } 'collapse' { $el = Find-One $winEl $p = $null if ($el.TryGetCurrentPattern([System.Windows.Automation.ExpandCollapsePattern]::Pattern, [ref]$p)) { $p.Collapse() [void]$script:Results.Add(@{ ok = $true; action = 'collapse'; matched = (Get-ConditionText $el) }) } else { throw "Control does not support ExpandCollapsePattern." } } 'window_minimize' { $wp = $null if ($winEl.TryGetCurrentPattern([System.Windows.Automation.WindowPattern]::Pattern, [ref]$wp)) { $wp.SetWindowVisualState([System.Windows.Automation.WindowVisualState]::Minimized) [void]$script:Results.Add(@{ ok = $true; action = 'window_minimize' }) } else { throw "Window does not support WindowPattern." } } 'window_maximize' { $wp = $null if ($winEl.TryGetCurrentPattern([System.Windows.Automation.WindowPattern]::Pattern, [ref]$wp)) { $wp.SetWindowVisualState([System.Windows.Automation.WindowVisualState]::Maximized) [void]$script:Results.Add(@{ ok = $true; action = 'window_maximize' }) } else { throw "Window does not support WindowPattern." } } 'window_restore' { $wp = $null if ($winEl.TryGetCurrentPattern([System.Windows.Automation.WindowPattern]::Pattern, [ref]$wp)) { $wp.SetWindowVisualState([System.Windows.Automation.WindowVisualState]::Normal) [void]$script:Results.Add(@{ ok = $true; action = 'window_restore' }) } else { throw "Window does not support WindowPattern." } } 'window_close' { $wp = $null if ($winEl.TryGetCurrentPattern([System.Windows.Automation.WindowPattern]::Pattern, [ref]$wp)) { $wp.Close() [void]$script:Results.Add(@{ ok = $true; action = 'window_close' }) } else { throw "Window does not support WindowPattern." } } 'window_move' { if ($null -eq $X -or $null -eq $Y) { throw "window_move requires x and y." } $wp = $null if ($winEl.TryGetCurrentPattern([System.Windows.Automation.WindowPattern]::Pattern, [ref]$wp)) { $wp.Move($X, $Y) [void]$script:Results.Add(@{ ok = $true; action = 'window_move'; x = $X; y = $Y }) } else { throw "Window does not support WindowPattern." } } default { throw "Unknown action '$Action'." } } Write-Output (ConvertTo-Json -Depth 8 -Compress @{ ok = $true; action = $Action; count = $script:Results.Count; items = $script:Results }) } catch { Write-Output (ConvertTo-Json -Depth 8 -Compress @{ ok = $false; error = $_.Exception.Message }) } `; async function runPs(pi: ExtensionAPI, script: string, args: string[], signal?: AbortSignal): Promise<{ stdout: string; stderr: string; code: number }> { const tmpFile = join(tmpdir(), `pi-ui-${Date.now()}-${Math.floor(Math.random() * 1e6)}.ps1`); await writeFile(tmpFile, script, "utf8"); try { const psArgs = ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", tmpFile, ...args]; const result = await pi.exec("powershell.exe", psArgs, { signal, timeout: 60000 }); return result; } finally { try { await unlink(tmpFile); } catch { /* ignore */ } } } function describeResult(parsed: any): { content: string; details: any } { if (!parsed) return { content: "No output from UI Automation.", details: {} }; if (!parsed.ok) { return { content: `UI Automation error: ${parsed.error}`, details: { ok: false, error: parsed.error } }; } const items = parsed.items || []; let content: string; if (parsed.action === "list_windows") { content = items.map((w: any) => `- ${w.title} (pid=${w.pid}, handle=${w.handle})`).join("\n") || "(no windows)"; } else if (items.length === 1 && items[0]?.ok === true && (parsed.action === "click" || parsed.action === "focus" || parsed.action === "toggle" || parsed.action === "set_range" || parsed.action === "get_range" || parsed.action === "select" || parsed.action === "scroll" || parsed.action === "expand" || parsed.action === "collapse")) { const e = items[0]; const bits: string[] = []; if (e.matched) bits.push(e.matched); if (e.action) bits.push(`did: ${e.action}`); if (e.value != null) bits.push(`value=${e.value}`); if (e.minimum != null && e.maximum != null) bits.push(`range=[${e.minimum}, ${e.maximum}]`); if (e.horizontal != null || e.vertical != null) bits.push(`scroll(h=${e.horizontal}, v=${e.vertical})`); if (e.before != null) bits.push(`was=${e.before}`); content = bits.join(" ") || `ok (${parsed.action})`; } else if (parsed.action === "click_coords" || parsed.action === "window_move" || parsed.action === "window_minimize" || parsed.action === "window_maximize" || parsed.action === "window_restore" || parsed.action === "window_close") { const e = items[0] || parsed; content = `${parsed.action} ok` + (e.x != null ? ` @(${e.x},${e.y})` : "") + (e.count ? ` x${e.count}` : ""); } else if (parsed.action === "get_value" || parsed.action === "get_text") { const e = items[0] || {}; content = e.note ? `value: ${e.value ?? e.text} (${e.note})` : `value: ${e.value ?? e.text}`; if (e.matched) content += ` | ${e.matched}`; } else { // find / list_controls / set_value / get_value etc. const lines = items.map((e: any) => { const parts: string[] = []; if (e.name) parts.push(e.name); if (e.type) parts.push(`[${e.type}]`); if (e.id) parts.push(`id=${e.id}`); if (e.value != null) parts.push(`value=${e.value}`); if (e.text != null && e.text !== e.name) parts.push(`text=${e.text}`); if (e.enabled === false) parts.push("disabled"); if (e.patterns && e.patterns.length) parts.push(`(patterns=${e.patterns.join(",")})`); if (e.matched) parts.push(e.matched); return "- " + parts.join(" "); }); content = lines.join("\n") || "(no controls matched)"; } return { content, details: { ok: true, count: parsed.count, action: parsed.action } }; } export function registerUiAutomation(pi: ExtensionAPI) { if (!IS_WINDOWS) { pi.registerTool({ name: "ui_automation", label: "UI Automation", description: "Not available on this platform (Windows only).", parameters: Type.Object({}), async execute() { return { content: [{ type: "text", text: "ui_automation is not available on non-Windows platforms." }], details: {}, }; }, }); return; } pi.registerTool({ name: "ui_automation", label: "UI Automation", description: "Read and operate UI controls of a Windows program via System.Windows.Automation. Control actions: list_windows, list_controls, find, click (invoke), click_coords (mouse click; absolute screen or relative-to-window when relative=true), set_value, get_value, get_text, focus, toggle, set_range/get_range, select, scroll, expand, collapse. Window actions: window_minimize, window_maximize, window_restore, window_close, window_move. Target a window by process (name/PID), window_title, or window_handle (hex). Match controls by name, automation_id, or control_type.", promptSnippet: "Read or operate native Windows UI controls (buttons, edit boxes, menus, checkboxes) of a program via UIAutomation", parameters: Params, async execute(_toolCallId, params, signal) { if (signal?.aborted) return { content: [{ type: "text", text: "Cancelled" }], details: {} }; const args = [ params.action, params.process ?? "", params.window_title ?? "", params.window_handle ?? "", params.name ?? "", params.automation_id ?? "", params.control_type ?? "", params.value ?? "", (params.value2 ?? "").toString(), (params.x ?? "").toString(), (params.y ?? "").toString(), params.button ?? "", (params.click_count ?? 0).toString(), (params.amount ?? 0).toString(), params.scroll_axis ?? "", (params.max_results ?? 100).toString(), (params.relative ? "true" : "false"), params.relative_to ?? "", ]; const result = await runPs(pi, UI_SCRIPT, args, signal); if (result.code !== 0) { return { content: [{ type: "text", text: `PowerShell failed (${result.code}): ${result.stderr || result.stdout}` }], details: { ok: false, code: result.code }, }; } let parsed: any; try { parsed = JSON.parse(result.stdout); } catch (e: any) { return { content: [{ type: "text", text: `Unparseable output: ${result.stdout}` }], details: { ok: false, error: e.message }, }; } const d = describeResult(parsed); return { content: [{ type: "text", text: d.content }], details: d.details }; }, }); }