/** * Computer-use payload types — wire format for `computer-use-v1.md`. * * A motebit with a desktop surface (Tauri) can observe and act on the user's * computer: screenshot, move, click, type, scroll. This spec pins the wire * format of each action and observation so the signed audit trail is * implementation-agnostic — a non-TypeScript verifier can validate payloads * against the committed JSON Schema rather than TypeScript's structural view. * * Shape: actions are a **nested discriminated union** (`action: { kind, ... }`) * not a flat envelope. Impossible states are structurally unrepresentable — * drag-only fields never appear on a click, type-only fields never appear on * a scroll. Cross-language SDKs (Rust enums, Python tagged unions) map * directly; JSON Schema emits clean `oneOf` branches; model output stays * rigorous. * * Observation results carry **artifact references** (`artifact_id + * artifact_sha256`) not inline bytes. The receipt artifact store (see * `spec/execution-ledger-v1.md`) holds the bytes; the observation payload * binds to them by hash. Keeps signed receipts O(bytes of metadata) instead * of O(bytes of image). Git / IPFS / S3+hash pattern. * * Every type named here is referenced by a `### X.Y — Name` section under a * `#### Wire format (foundation law)` block in `spec/computer-use-v1.md`, * so `check-spec-coverage` (invariant #9) keeps the spec and types in * lockstep. * * ── Release status ─────────────────────────────────────────────────── * * Every export in this file carries `@alpha` — the wire format was revised * on 2026-04-22 in response to external principal review (commit 54158b11, * flat envelope → nested discriminated union) and has only soaked inside * the monorepo since. `@alpha` carves these types out of `@motebit/protocol`'s * SemVer contract for the 1.x series: the `check-api-surface` baseline still * tracks their shape so unintended drift is caught, but a shape reshape does * not force a major bump. Promote to `@beta` → `@public` once a second * producer (cloud-browser surface, federation peer replaying an audit log) * or a non-desktop-shim consumer exercises the format in anger. */ import type { ControlState } from "./co-browse.js"; /** * A point in primary-display logical pixel coordinates. `(0, 0)` = top-left. * @alpha */ export interface ComputerPoint { readonly x: number; readonly y: number; } /** * Optional semantic target information attached to pointer actions. When * available from the accessibility layer or DOM, it lets verifiers and * approval UX explain "motebit clicked the Send button" instead of only * "(512, 384)". Execution still happens at the pixel coordinates in * `target`; the hint is advisory. * @alpha */ export interface ComputerTargetHint { /** Role name. Examples: `"button"`, `"link"`, `"textbox"`, `"menuitem"`. */ readonly role?: string; /** Accessible label or visible text. */ readonly label?: string; /** * Where the hint came from. `"accessibility"` (OS AX API), `"dom"` * (browser inspection), `"vision"` (OCR/ML), `"user_annotation"` (user * labeled the region in-session). */ readonly source: string; } /** * Capture the primary display's current frame. * @alpha */ export interface ScreenshotAction { readonly kind: "screenshot"; } /** * Read the current cursor coordinates. * @alpha */ export interface CursorPositionAction { readonly kind: "cursor_position"; } /** * Single mouse click at `target`. * @alpha */ export interface ClickAction { readonly kind: "click"; readonly target: ComputerPoint; /** `"left" | "right" | "middle"`. Defaults to `"left"`. */ readonly button?: string; /** Subset of `["cmd", "ctrl", "alt", "shift"]`. */ readonly modifiers?: readonly string[]; readonly target_hint?: ComputerTargetHint; } /** * Two clicks in rapid succession at `target`. * @alpha */ export interface DoubleClickAction { readonly kind: "double_click"; readonly target: ComputerPoint; readonly button?: string; readonly modifiers?: readonly string[]; readonly target_hint?: ComputerTargetHint; } /** * Move cursor to `target` without clicking. * @alpha */ export interface MouseMoveAction { readonly kind: "mouse_move"; readonly target: ComputerPoint; readonly target_hint?: ComputerTargetHint; } /** * Press at `from`, move to `to`, release. * @alpha */ export interface DragAction { readonly kind: "drag"; readonly from: ComputerPoint; readonly to: ComputerPoint; readonly button?: string; readonly modifiers?: readonly string[]; /** Total drag duration in ms. Implementation may interpolate intermediate points. */ readonly duration_ms?: number; readonly target_hint?: ComputerTargetHint; } /** * Keyboard text input. * @alpha */ export interface TypeAction { readonly kind: "type"; readonly text: string; /** * Per-character delay in ms. Omitted = implementation default. Set to a * stable value for deterministic replay of typing cadence. */ readonly per_char_delay_ms?: number; } /** * Keyboard combination. Example: `"cmd+c"`, `"ctrl+shift+t"`, `"escape"`. * @alpha */ export interface KeyAction { readonly kind: "key"; readonly key: string; } /** * Scroll at `target` by `(dx, dy)` wheel deltas. * @alpha */ export interface ScrollAction { readonly kind: "scroll"; readonly target: ComputerPoint; readonly dx: number; readonly dy: number; } /** * Navigate the active browser context to `url`. Cloud-browser-only in * v1: the headless Playwright runtime in `services/browser-sandbox` * has no address-bar UI for `key`/`type` to drive, so the spec * promotion path * (`spec/computer-use-v1.md` §"No new wire-format actions… real-usage- * driven, not speculative") fired when the AI hit "navigate to * tesla.com" against the cloud-browser dispatcher. * * Desktop dispatcher (`apps/desktop/src-tauri/src/computer_use.rs`) * does NOT implement this action — OS-level computer-use has no * notion of "the active browser context"; the user is in control of * which app is focused. The dispatcher-parity check at * `scripts/check-computer-use-dispatcher-parity.ts` carries an * ALLOWLIST entry naming desktop as deferred until an OS-level * navigation use-case proves itself. * @alpha */ export interface NavigateAction { readonly kind: "navigate"; /** * Target URL. Implementations SHOULD normalize relative-looking * inputs (`example.com` → `https://example.com`) but MAY reject * malformed inputs with a `not_supported` failure. */ readonly url: string; } /** * element-1 — click on a structurally-addressed element. Server * resolves `element_id` (issued by a prior `read_page` extraction * via the stamped `data-motebit-id` attribute), scrolls into view, * clicks the center of the element's bounding rect. * * Prefer this over coordinate-based `click` when the target was * discovered via `read_page` — durable against viewport, zoom, and * layout shifts. Coordinate `click` remains for purely-visual tasks * (clicking on a position seen in pixels). * * On staleness (page navigated since read_page, page reloaded, * element removed by JS) the result fails with `element_not_found` * and the AI re-reads to refresh the id space. * @alpha */ export interface ClickElementAction { readonly kind: "click_element"; readonly element_id: string; } /** * element-1 — focus a structurally-addressed element without * clicking. Useful for setting up `type` calls without the side- * effects of a click (some fields open dropdowns / dialogs on * click; focus-only avoids them). * @alpha */ export interface FocusElementAction { readonly kind: "focus_element"; readonly element_id: string; } /** * element-1 — semantic-intent type. Server focuses the addressed * element first (so keystrokes can't be swallowed by the page's * focus state), optionally clears existing value, then types. The * result envelope carries `text_appeared` truth-feedback so the AI * doesn't confabulate "I typed it" when the keystrokes went * nowhere. * * Composes the two-step click+type workflow into one semantic * action — closes the action-truth gap from the witnessed * 2026-05-08 smoke (AI typed without focus → keystrokes * swallowed → AI claimed success). * @alpha */ export interface TypeIntoAction { readonly kind: "type_into"; readonly element_id: string; readonly text: string; /** Per-character delay in ms; same shape as the lower-level `type` action. */ readonly per_char_delay_ms?: number; /** * If true (default), the field's current value is cleared before * typing. Mirrors the human "type fresh into this box" intent. Set * to false to append to existing text. */ readonly clear_first?: boolean; } /** * Full action taxonomy. Every concrete action the motebit can request is * one of these. Exhaustive discriminated union on `kind`. * @alpha */ export type ComputerAction = ScreenshotAction | CursorPositionAction | ClickAction | DoubleClickAction | MouseMoveAction | DragAction | TypeAction | KeyAction | ScrollAction | NavigateAction | ClickElementAction | FocusElementAction | TypeIntoAction; /** * Discriminator values — useful for JSON Schema enum and runtime validation. * @alpha */ export declare const COMPUTER_ACTION_KINDS: readonly ["screenshot", "cursor_position", "click", "double_click", "mouse_move", "drag", "type", "key", "scroll", "navigate", "click_element", "focus_element", "type_into"]; /** @alpha */ export type ComputerActionKind = (typeof COMPUTER_ACTION_KINDS)[number]; /** * One invocation of the `computer` tool. The `action` field holds a nested * discriminated variant; all action-specific fields live inside that * object. Impossible states are structurally unrepresentable. * @alpha */ export interface ComputerActionRequest { /** Open session the action belongs to. */ readonly session_id: string; readonly action: ComputerAction; } /** * Outcome of a successful observation action's execution. Returned as the * `data` field of a `ToolResult`. Discriminated by `kind`. * @alpha */ export type ComputerObservationResult = ScreenshotObservation | CursorPositionObservation; /** * Structured redaction metadata. Replaces a bare boolean — a verifier can * now prove *what* was redacted, under *which* policy version, and whether * the bytes the AI saw are raw or a projection. * @alpha */ export interface ComputerRedaction { /** `true` iff any region was classified as sensitive and masked. */ readonly applied: boolean; /** * Projection shape of the bytes the AI consumed. `"raw"` = unmodified * capture, `"masked"` = regions replaced with solid fill, * `"blurred"` = regions convolved, `"cropped"` = sensitive regions * removed from frame. Surfaces MAY add projection kinds in v1.1. */ readonly projection_kind: string; /** Version string of the sensitivity-classification policy that ran. */ readonly policy_version?: string; /** Number of regions classified as sensitive in the raw frame. */ readonly classified_regions_count?: number; /** * SHA-256 digest of a canonical JSON array of classified regions * (each region: `{ x, y, w, h, classification }`). Lets a verifier * replay what was masked without exposing the region list in the * receipt if policy dictates. */ readonly classified_regions_digest?: string; } /** * Screenshot observation. Bytes live in the receipt artifact store keyed * by `artifact_id` and bound by `artifact_sha256`. When redaction produces * a distinct projection (e.g. masked image), both the raw and projection * artifact IDs are referenced so a verifier can fetch either depending on * authorization. * @alpha */ export interface ScreenshotObservation { readonly kind: "screenshot"; readonly session_id: string; /** Artifact ID of the raw capture in the artifact store. */ readonly artifact_id: string; /** SHA-256 of the raw capture's bytes. */ readonly artifact_sha256: string; /** `"png" | "jpeg"`. */ readonly image_format: string; /** Image width in logical pixels. */ readonly width: number; /** Image height in logical pixels. */ readonly height: number; /** Unix ms of the capture. */ readonly captured_at: number; readonly redaction: ComputerRedaction; /** * Artifact ID of the redacted projection, when `redaction.applied` and * the projection differs from the raw capture. When absent, the AI saw * the raw bytes (redaction.projection_kind === "raw"). */ readonly projection_artifact_id?: string; /** SHA-256 of the projection bytes. Paired with `projection_artifact_id`. */ readonly projection_artifact_sha256?: string; } /** * Cursor-position observation — single coordinate pair. * @alpha */ export interface CursorPositionObservation { readonly kind: "cursor_position"; readonly session_id: string; readonly x: number; readonly y: number; readonly captured_at: number; } /** * A single heading extracted from the page's DOM. `level` is the * HTML heading level 1-6; `text` is the visible text content. The * server returns headings in document order so the AI can rebuild * the page's outline without parsing markup. * @alpha */ export interface ReadPageHeading { readonly level: number; readonly text: string; } /** * A single visible link extracted from the page's DOM. `text` is * the anchor's visible label (innerText); `href` is the absolute * URL the link points to. Server skips empty-label links and * `javascript:` / fragment-only hrefs. * @alpha */ export interface ReadPageLink { readonly text: string; readonly href: string; } /** * A single typeable input extracted from the page's DOM — * `` (text-shaped types: text, search, email, url, tel, * number, password) and `