import type { AgentToolResult } from "@earendil-works/pi-agent-core"; import { Text } from "@earendil-works/pi-tui"; import type { ExtensionContext, ToolDefinition } from "@earendil-works/pi-coding-agent"; import { Type, type Static } from "typebox"; import { toolCallLine, toolResultLine, resultSummary } from "pi-cockpit/src/quiet-tools.ts"; import { computerUseManager, type ComputerUseManagerLike, type ComputerUseStatus, type ManagerOperationOptions, } from "./computer-use/manager.ts"; import { ComputerUseError, errorInfo, type CapturedFrame, type PhysicalRect } from "./computer-use/types.ts"; import { getSopRegistry, SOP_INDEX_EXTRAS, SOP_INDEX_HEADERS } from "./sop/sop-registry-singleton.ts"; const ACTIONS = [ "guide", "capabilities", "status", "permissions", "list_windows", "activate", "screenshot", "ocr", "detect", "ui_tree", "find_control", "press_control", "click", "double_click", "right_click", "move", "scroll", "drag", "press", "type", "paste", "find_block", ] as const; type ComputerUseAction = typeof ACTIONS[number]; const ComputerUseAction = Type.Unsafe({ type: "string", enum: [...ACTIONS], description: "Desktop action. Call guide first, then use observe-act-verify. Mouse actions are click, double_click, right_click, move, scroll, and drag.", }); const Source = Type.Unsafe<"screen" | "window" | "region" | "image">({ type: "string", enum: ["screen", "window", "region", "image"] }); const Region = Type.Object({ x: Type.Number({ description: "Physical screen x origin" }), y: Type.Number({ description: "Physical screen y origin" }), width: Type.Number({ minimum: 1 }), height: Type.Number({ minimum: 1 }), }, { additionalProperties: false }); /** The broad object is paired with action-specific allOf requirements and runtime checks below. */ export const ComputerUseParams = Type.Object({ action: ComputerUseAction, topic: Type.Optional(Type.String({ description: "SOP topic for guide; omit for the registry index" })), source: Type.Optional(Source), path: Type.Optional(Type.String({ minLength: 1, description: "PNG path for image vision input" })), template_path: Type.Optional(Type.String({ minLength: 1, description: "PNG template path for find_block" })), window_id: Type.Optional(Type.String({ minLength: 1, description: "Target window id from list_windows" })), display_id: Type.Optional(Type.String()), region: Type.Optional(Region), visible_only: Type.Optional(Type.Boolean()), title: Type.Optional(Type.String()), app: Type.Optional(Type.String()), query: Type.Optional(Type.String({ minLength: 1, description: "Control query for find_control" })), control_ref: Type.Optional(Type.String({ minLength: 1, description: "Fresh control ref from ui_tree/find_control" })), max_depth: Type.Optional(Type.Integer({ minimum: 0, maximum: 100 })), include_offscreen: Type.Optional(Type.Boolean()), enabled_only: Type.Optional(Type.Boolean()), max_results: Type.Optional(Type.Integer({ minimum: 1, maximum: 500 })), x: Type.Optional(Type.Number({ description: "Physical x or client-relative x" })), y: Type.Optional(Type.Number({ description: "Physical y or client-relative y" })), to_x: Type.Optional(Type.Number()), to_y: Type.Optional(Type.Number()), coordinate_space: Type.Optional(Type.Unsafe<"screen_physical" | "window_client_physical">({ type: "string", enum: ["screen_physical", "window_client_physical"] })), allow_outside_window: Type.Optional(Type.Boolean()), allow_destructive: Type.Optional(Type.Boolean()), target_context: Type.Optional(Type.Unsafe<"desktop" | "local_game" | "network_game">({ type: "string", enum: ["desktop", "local_game", "network_game"] })), duration_ms: Type.Optional(Type.Number({ minimum: 0, maximum: 120_000 })), observe_after: Type.Optional(Type.Boolean({ description: "For mouse actions, attach a fresh target-window screenshot after the action (default true)" })), direction: Type.Optional(Type.Unsafe<"up" | "down" | "left" | "right">({ type: "string", enum: ["up", "down", "left", "right"], description: "Scroll direction" })), magnitude: Type.Optional(Type.Integer({ minimum: 1, maximum: 10_000, description: "Bounded scroll amount" })), keys: Type.Optional(Type.Array(Type.String({ minLength: 1 }), { minItems: 1, maxItems: 64 })), text: Type.Optional(Type.String({ maxLength: 100_000 })), interval_ms: Type.Optional(Type.Number({ minimum: 0, maximum: 10_000 })), enhance: Type.Optional(Type.Boolean()), langs: Type.Optional(Type.Array(Type.String({ minLength: 1 }), { maxItems: 32 })), mode: Type.Optional(Type.Unsafe<"match" | "crop">({ type: "string", enum: ["match", "crop"] })), confidence: Type.Optional(Type.Number({ minimum: 0, maximum: 1 })), iou_threshold: Type.Optional(Type.Number({ minimum: 0, maximum: 1 })), threshold: Type.Optional(Type.Number({ minimum: 0, maximum: 1 })), timeout_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 300_000, description: "Bounded operation timeout in milliseconds" })), }, { additionalProperties: false, allOf: [ { if: { properties: { action: { const: "guide" } }, required: ["action"] }, then: { not: { required: ["source"] } } }, { if: { properties: { action: { enum: ["screenshot", "ocr", "detect", "find_block"] } }, required: ["action"] }, then: { required: ["source"] } }, { if: { properties: { action: { enum: ["screenshot", "find_block"] } }, required: ["action"] }, then: { not: { properties: { source: { const: "image" } } } } }, { if: { properties: { source: { const: "window" } }, required: ["source"] }, then: { required: ["window_id"] } }, { if: { properties: { source: { const: "region" } }, required: ["source"] }, then: { required: ["region"] } }, { if: { properties: { source: { const: "image" } }, required: ["source"] }, then: { required: ["path"] } }, { if: { properties: { action: { enum: ["activate", "ui_tree", "find_control", "press_control", "click", "double_click", "right_click", "move", "scroll", "drag", "press", "type", "paste"] } }, required: ["action"] }, then: { required: ["window_id"] } }, { if: { properties: { action: { enum: ["click", "double_click", "right_click", "move", "scroll", "drag"] } }, required: ["action"] }, then: { required: ["x", "y"] } }, { if: { properties: { action: { const: "scroll" } }, required: ["action"] }, then: { required: ["direction", "magnitude"] } }, { if: { properties: { action: { const: "drag" } }, required: ["action"] }, then: { required: ["to_x", "to_y"] } }, { if: { properties: { action: { const: "press_control" } }, required: ["action"] }, then: { required: ["control_ref"] } }, { if: { properties: { action: { const: "find_control" } }, required: ["action"] }, then: { required: ["query"] } }, { if: { properties: { action: { const: "press" } }, required: ["action"] }, then: { required: ["keys"] } }, { if: { properties: { action: { enum: ["type", "paste"] } }, required: ["action"] }, then: { required: ["text"] } }, { if: { properties: { action: { const: "find_block" } }, required: ["action"] }, then: { required: ["template_path"] } }, ], } as Record); // Action-specific requirements (source/window_id per source kind, x/y per // pointer action, query/keys/text/template_path per action) are enforced at // runtime by validateComputerUseParams() in execute(), not in the model-facing // schema. The host validates against the full ComputerUseParams; the // model-facing declaration must omit conditionals (allOf/if/then/not), which // Gemini and xAI function declarations reject with 400. Same precedent as // TodoModelParams in src/extension/schemas.ts. const { allOf: _computerUseConditions, ...computerUseModelSchema } = ComputerUseParams as unknown as Record; export const ComputerUseModelParams = Type.Unsafe(computerUseModelSchema); export type ComputerUseParamsType = Static; export interface ComputerUseToolDetails { action: ComputerUseAction; result?: unknown; error?: ReturnType; image?: { mimeType: string; width: number; height: number; bytes: number; source: string }; } const POINTER_ACTIONS = new Set(["click", "double_click", "right_click", "move", "scroll", "drag"]); const VISION_ACTIONS = new Set(["ocr", "detect"]); const SOURCE_ACTIONS = new Set(["screenshot", "ocr", "detect", "find_block"]); export type ComputerUseToolDefinition = ToolDefinition & { execute: ToolDefinition["execute"] & ( (id: string, params: never, signal: AbortSignal, onUpdate: undefined, ctx: ExtensionContext) => Promise> ); }; export function createComputerUseTool(manager: ComputerUseManagerLike = computerUseManager): ComputerUseToolDefinition { return { name: "computer_use", label: "Computer Use", description: "Observe and control the physical desktop through a serialized, fail-closed ComputerUseManager. Use computer_use { action: \"guide\" } before desktop operations. Coordinates are physical pixels; activate and verify foreground state, then observe-act-verify. Mouse actions attach a fresh target-window screenshot by default so the next action uses current visual state. Near-zero diagnostics latch input until a fresh probe. Wayland, permissions, network-game software input, and unavailable native providers remain restricted.", promptSnippet: "Use computer_use for bounded physical desktop observation and input. Call action=guide first, then capabilities/permissions and an observe action. Use verified mouse coordinates for visual targets instead of avoiding the pointer with keyboard navigation. Mouse actions observe the target window again by default. Stop on near-zero, timeout, permission, Wayland, stale-control, or foreground errors.", promptGuidelines: [ "Before desktop operations, call computer_use action=guide; load core for the observe-act-verify loop, pointer for the mouse recipe, coordinates for physical/client-origin/DPI rules, safety for destructive and near-zero stops, and platform for permissions/Wayland/provider limits.", "Use capabilities and permissions before assuming screen capture, accessibility, input, or window control. Observe through an ordered fallback: list_windows, then ui_tree/find_control when reliable, then screenshot with detect/OCR; use the model only to interpret verified pixels, never to invent coordinates. Activate before input.", "When a target is visually identifiable but has no reliable accessibility control, use click/double_click/right_click with the verified box center; use move for hover UI, scroll at a verified point for bounded viewport movement, and drag for sliders, canvas, or reordering. Do not substitute repeated Tab/arrow navigation merely to avoid using the mouse.", "Map screenshot/OCR/detect coordinates through image.origin into screen_physical coordinates, or use verified window_client_physical coordinates. Never guess a point or DPI scale.", "Mouse actions attach a fresh target-window screenshot by default. Treat it as the next observation before choosing another action; set observe_after=false only when an immediate visual observation is intentionally unnecessary.", "Coordinates are physical pixels. window_client_physical is resolved from the verified client origin (ClientToScreen), never the outer window bounds. Windows DPI and macOS Retina scaling can change logical coordinates.", "Never retry a near-zero, timeout, foreground, stale-control, permission, Wayland, or network-game failure blindly. Destructive input requires allow_destructive=true.", ], parameters: ComputerUseModelParams, executionMode: "sequential", async execute(_id, rawParams, signal, _onUpdate, ctx): Promise> { const validation = validateComputerUseParams(rawParams as unknown); if (validation) return failure("unknown", validation); const params = rawParams as ComputerUseParamsType; const action = params.action as ComputerUseAction; try { if (action === "guide") { const registry = getSopRegistry(ctx.cwd); await registry.ensureLoaded(); const topic = stringValue(params.topic); if (!topic) { const text = registry.renderIndex("computer_use", SOP_INDEX_HEADERS.computer_use, SOP_INDEX_EXTRAS.computer_use); return success(text, { action, result: text }); } const doc = registry.get("computer_use", topic); if (!doc) return failure(action, `Unknown SOP topic ${JSON.stringify(topic)}. Available: ${registry.topics("computer_use").join(", ")}.`); return success(doc.body, { action, result: doc.body }); } const options: ManagerOperationOptions = params.timeout_ms === undefined ? { signal } : { signal, deadline: Date.now() + params.timeout_ms }; const finishPointer = async (operation: Promise) => pointerResult(action, await operation, params.window_id!, params.observe_after !== false, manager, options); let result: unknown; switch (action) { case "capabilities": result = await manager.capabilities(options); break; case "status": result = await manager.status(options); break; case "permissions": result = await manager.permissions(options); break; case "list_windows": result = await manager.listWindows({ visibleOnly: params.visible_only, title: params.title, app: params.app }, options); break; case "activate": result = await manager.activate(params.window_id!, options); break; case "screenshot": { const frame = await manager.screenshot({ source: params.source as "screen" | "window" | "region", window_id: params.window_id, region: params.region as PhysicalRect | undefined, display_id: params.display_id, ...options }); return frameResult(action, frame); } case "ocr": result = await manager.ocr({ source: params.source as "image" | "screen" | "window" | "region", path: params.path, window_id: params.window_id, region: params.region as PhysicalRect | undefined, display_id: params.display_id, enhance: params.enhance, langs: params.langs, mode: params.mode, confidence: params.confidence, iou_threshold: params.iou_threshold, ...options }); break; case "detect": result = await manager.detect({ source: params.source as "image" | "screen" | "window" | "region", path: params.path, window_id: params.window_id, region: params.region as PhysicalRect | undefined, display_id: params.display_id, enhance: params.enhance, langs: params.langs, mode: params.mode, confidence: params.confidence, iou_threshold: params.iou_threshold, ...options }); break; case "ui_tree": result = await manager.uiTree({ windowId: params.window_id!, maxDepth: params.max_depth, includeOffscreen: params.include_offscreen, ...options }); break; case "find_control": result = await manager.findControl({ windowId: params.window_id!, query: params.query!, enabledOnly: params.enabled_only, maxResults: params.max_results, ...options }); break; case "press_control": result = await manager.pressControl({ window_id: params.window_id!, control_ref: params.control_ref!, allow_destructive: params.allow_destructive, target_context: params.target_context, ...options }); break; case "click": return await finishPointer(manager.click({ window_id: params.window_id!, x: params.x!, y: params.y!, coordinate_space: params.coordinate_space, allow_outside_window: params.allow_outside_window, allow_destructive: params.allow_destructive, target_context: params.target_context, duration_ms: params.duration_ms, ...options })); case "double_click": return await finishPointer(manager.doubleClick({ window_id: params.window_id!, x: params.x!, y: params.y!, coordinate_space: params.coordinate_space, allow_outside_window: params.allow_outside_window, allow_destructive: params.allow_destructive, target_context: params.target_context, duration_ms: params.duration_ms, ...options })); case "right_click": return await finishPointer(manager.rightClick({ window_id: params.window_id!, x: params.x!, y: params.y!, coordinate_space: params.coordinate_space, allow_outside_window: params.allow_outside_window, allow_destructive: params.allow_destructive, target_context: params.target_context, duration_ms: params.duration_ms, ...options })); case "move": return await finishPointer(manager.move({ window_id: params.window_id!, x: params.x!, y: params.y!, coordinate_space: params.coordinate_space, allow_outside_window: params.allow_outside_window, allow_destructive: params.allow_destructive, target_context: params.target_context, duration_ms: params.duration_ms, ...options })); case "scroll": { if (!manager.scroll) throw new ComputerUseError({ code: "DEPENDENCY_UNAVAILABLE", message: "The injected computer-use manager does not support scroll", capability: "input", retryable: false }); return await finishPointer(manager.scroll({ window_id: params.window_id!, x: params.x!, y: params.y!, coordinate_space: params.coordinate_space, allow_outside_window: params.allow_outside_window, allow_destructive: params.allow_destructive, target_context: params.target_context, direction: params.direction!, magnitude: params.magnitude!, ...options })); } case "drag": return await finishPointer(manager.drag({ window_id: params.window_id!, x: params.x!, y: params.y!, to_x: params.to_x!, to_y: params.to_y!, coordinate_space: params.coordinate_space, allow_outside_window: params.allow_outside_window, allow_destructive: params.allow_destructive, target_context: params.target_context, duration_ms: params.duration_ms, ...options })); case "press": result = await manager.press({ window_id: params.window_id!, keys: params.keys!, allow_destructive: params.allow_destructive, target_context: params.target_context, ...options }); break; case "type": result = await manager.type({ window_id: params.window_id!, text: params.text!, interval_ms: params.interval_ms, allow_destructive: params.allow_destructive, target_context: params.target_context, ...options }); break; case "paste": result = await manager.paste({ window_id: params.window_id!, text: params.text!, interval_ms: params.interval_ms, allow_destructive: params.allow_destructive, target_context: params.target_context, ...options }); break; case "find_block": result = await manager.findBlock({ template_path: params.template_path!, source: params.source as "screen" | "window" | "region", window_id: params.window_id, region: params.region as PhysicalRect | undefined, threshold: params.threshold, ...options }); break; default: return failure(action, `Unsupported computer_use action ${JSON.stringify(action)}.`); } if (isFailureEnvelope(result)) return failure(action, result.error); const text = formatResult(result); return success(text, { action, result }); } catch (error) { return failure(action, errorInfo(error)); } }, renderShell: "self", renderCall(args, theme, ctx) { if (ctx?.isPartial === false) return new Text("", 0, 0); const action = String(args.action ?? "?"); const target = args.window_id ? ` ${String(args.window_id).slice(0, 32)}` : ""; return toolCallLine(theme, "computer_use", `${action}${target}`); }, renderResult(result, opts, theme, ctx) { if (opts.isPartial) return new Text("", 0, 0); const action = String(ctx.args.action ?? "?"); const details = result.details as ComputerUseToolDetails | undefined; const isError = ctx.isError || (result as { isError?: boolean }).isError === true; const detail = result.content.filter((item) => item.type === "text").map((item) => "text" in item ? item.text : "").join("\n"); return toolResultLine(theme, { name: "computer_use", ok: !isError, arg: action, summary: resultSummary(result), expanded: opts.expanded, detail }); }, }; } function frameResult(action: ComputerUseAction, frame: CapturedFrame): AgentToolResult { const image = { type: "image" as const, data: Buffer.from(frame.bytes).toString("base64"), mimeType: frame.image.mimeType }; const details = { action, image: { mimeType: frame.image.mimeType, width: frame.image.width, height: frame.image.height, bytes: frame.bytes.byteLength, source: frame.image.source }, result: frame.image }; return { content: [image, { type: "text", text: JSON.stringify(frame.image) }], details } as AgentToolResult; } async function pointerResult( action: ComputerUseAction, actionResult: unknown, windowId: string, observeAfter: boolean, manager: ComputerUseManagerLike, options: ManagerOperationOptions, ): Promise> { if (!observeAfter) return success(formatResult(actionResult), { action, result: actionResult }); try { const frame = await manager.screenshot({ source: "window", window_id: windowId, ...options }); const result = { action_result: actionResult, observation: frame.image }; const image = { type: "image" as const, data: Buffer.from(frame.bytes).toString("base64"), mimeType: frame.image.mimeType }; const details = { action, image: { mimeType: frame.image.mimeType, width: frame.image.width, height: frame.image.height, bytes: frame.bytes.byteLength, source: frame.image.source }, result }; return { content: [image, { type: "text", text: formatResult(result) }], details } as AgentToolResult; } catch (error) { const result = { action_result: actionResult, observation: { ok: false, error: errorInfo(error) } }; return success(formatResult(result), { action, result }); } } function success(text: string, details: ComputerUseToolDetails): AgentToolResult { return { content: [{ type: "text", text }], details } as AgentToolResult; } function failure(action: string, error: unknown): AgentToolResult { const info = typeof error === "string" ? { code: "INTERNAL" as const, message: error, retryable: false } : isErrorInfoLike(error) ? error : errorInfo(error); return { content: [{ type: "text", text: JSON.stringify({ error: info }, null, 2) }], isError: true, details: { action: action as ComputerUseAction, error: info } } as AgentToolResult; } function isErrorInfoLike(value: unknown): value is ReturnType { if (!value || typeof value !== "object") return false; const record = value as Record; return typeof record.code === "string" && typeof record.message === "string" && typeof record.retryable === "boolean"; } function formatResult(value: unknown): string { const text = typeof value === "string" ? value : JSON.stringify(value, null, 2) ?? String(value); return text.length > 60_000 ? `${text.slice(0, 60_000)}\n[output truncated]` : text; } function isFailureEnvelope(value: unknown): value is { ok: false; error: unknown } { return Boolean(value && typeof value === "object" && (value as { ok?: unknown }).ok === false && "error" in value); } function stringValue(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } function validateComputerUseParams(value: unknown): string | undefined { if (!value || typeof value !== "object" || Array.isArray(value)) return "computer_use parameters must be an object."; const p = value as Record; if (typeof p.action !== "string" || !(ACTIONS as readonly string[]).includes(p.action)) return `action must be one of: ${ACTIONS.join(", ")}.`; const action = p.action; if (action === "guide") { if (p.source !== undefined) return "guide does not accept source; use topic to load a guide document."; return; } if (SOURCE_ACTIONS.has(action)) { if (!["screen", "window", "region", "image"].includes(String(p.source))) return `${action} requires source: screen, window, region, or image.`; if (p.source === "image" && !stringValue(p.path)) return `${action} with source=image requires path to a PNG image.`; if (p.source === "window" && !stringValue(p.window_id)) return `${action} with source=window requires window_id.`; if (p.source === "region" && !p.region) return `${action} with source=region requires region {x,y,width,height}.`; if (action === "screenshot" && p.source === "image") return "screenshot accepts source=screen, window, or region; use ocr/detect for an image file."; if (action === "find_block" && p.source === "image") return "find_block accepts screen, window, or region source; use ocr/detect for an image file."; } if (["activate", "ui_tree", "find_control", "press_control", ...POINTER_ACTIONS, "press", "type", "paste"].includes(action) && !stringValue(p.window_id)) return `${action} requires window_id from list_windows.`; if (POINTER_ACTIONS.has(action)) { if (!finiteNumber(p.x) || !finiteNumber(p.y)) return `${action} requires finite x and y coordinates.`; if (action === "scroll" && (!(["up", "down", "left", "right"] as const).includes(p.direction as never) || !Number.isSafeInteger(p.magnitude) || (p.magnitude as number) < 1 || (p.magnitude as number) > 10_000)) return "scroll requires direction up/down/left/right and integer magnitude from 1 to 10000."; if (action === "drag" && (!finiteNumber(p.to_x) || !finiteNumber(p.to_y))) return "drag requires finite to_x and to_y coordinates."; } if (action === "press_control" && !stringValue(p.control_ref)) return "press_control requires a fresh control_ref from ui_tree or find_control."; if (action === "find_control" && !stringValue(p.query)) return "find_control requires a non-empty query."; if (action === "press" && (!Array.isArray(p.keys) || p.keys.length === 0)) return "press requires a non-empty keys array."; if ((action === "type" || action === "paste") && typeof p.text !== "string") return `${action} requires text.`; if (action === "find_block" && !stringValue(p.template_path)) return "find_block requires template_path."; if (p.region !== undefined && !validRegion(p.region)) return "region requires finite x/y and positive finite width/height."; const timeout = p.timeout_ms; if (timeout !== undefined && (typeof timeout !== "number" || !Number.isInteger(timeout) || timeout < 1 || timeout > 300_000)) return "timeout_ms must be an integer from 1 to 300000 milliseconds."; return; } function finiteNumber(value: unknown): value is number { return typeof value === "number" && Number.isFinite(value); } function validRegion(value: unknown): value is PhysicalRect { if (!value || typeof value !== "object") return false; const p = value as Record; return finiteNumber(p.x) && finiteNumber(p.y) && finiteNumber(p.width) && finiteNumber(p.height) && p.width > 0 && p.height > 0; }