/**
* Computer-use payload types — wire format for `computer-use-v1.md`.
*
* A motebit with a desktop surface (Tauri) can observe and act on the user's
* computer: screenshot, move, click, type, scroll. This spec pins the wire
* format of each action and observation so the signed audit trail is
* implementation-agnostic — a non-TypeScript verifier can validate payloads
* against the committed JSON Schema rather than TypeScript's structural view.
*
* Shape: actions are a **nested discriminated union** (`action: { kind, ... }`)
* not a flat envelope. Impossible states are structurally unrepresentable —
* drag-only fields never appear on a click, type-only fields never appear on
* a scroll. Cross-language SDKs (Rust enums, Python tagged unions) map
* directly; JSON Schema emits clean `oneOf` branches; model output stays
* rigorous.
*
* Observation results carry **artifact references** (`artifact_id +
* artifact_sha256`) not inline bytes. The receipt artifact store (see
* `spec/execution-ledger-v1.md`) holds the bytes; the observation payload
* binds to them by hash. Keeps signed receipts O(bytes of metadata) instead
* of O(bytes of image). Git / IPFS / S3+hash pattern.
*
* Every type named here is referenced by a `### X.Y — Name` section under a
* `#### Wire format (foundation law)` block in `spec/computer-use-v1.md`,
* so `check-spec-coverage` (invariant #9) keeps the spec and types in
* lockstep.
*
* ── Release status ───────────────────────────────────────────────────
*
* Every export in this file carries `@alpha` — the wire format was revised
* on 2026-04-22 in response to external principal review (commit 54158b11,
* flat envelope → nested discriminated union) and has only soaked inside
* the monorepo since. `@alpha` carves these types out of `@motebit/protocol`'s
* SemVer contract for the 1.x series: the `check-api-surface` baseline still
* tracks their shape so unintended drift is caught, but a shape reshape does
* not force a major bump. Promote to `@beta` → `@public` once a second
* producer (cloud-browser surface, federation peer replaying an audit log)
* or a non-desktop-shim consumer exercises the format in anger.
*/
import type { ControlState } from "./co-browse.js";
/**
* A point in primary-display logical pixel coordinates. `(0, 0)` = top-left.
* @alpha
*/
export interface ComputerPoint {
readonly x: number;
readonly y: number;
}
/**
* Optional semantic target information attached to pointer actions. When
* available from the accessibility layer or DOM, it lets verifiers and
* approval UX explain "motebit clicked the Send button" instead of only
* "(512, 384)". Execution still happens at the pixel coordinates in
* `target`; the hint is advisory.
* @alpha
*/
export interface ComputerTargetHint {
/** Role name. Examples: `"button"`, `"link"`, `"textbox"`, `"menuitem"`. */
readonly role?: string;
/** Accessible label or visible text. */
readonly label?: string;
/**
* Where the hint came from. `"accessibility"` (OS AX API), `"dom"`
* (browser inspection), `"vision"` (OCR/ML), `"user_annotation"` (user
* labeled the region in-session).
*/
readonly source: string;
}
/**
* Capture the primary display's current frame.
* @alpha
*/
export interface ScreenshotAction {
readonly kind: "screenshot";
}
/**
* Read the current cursor coordinates.
* @alpha
*/
export interface CursorPositionAction {
readonly kind: "cursor_position";
}
/**
* Single mouse click at `target`.
* @alpha
*/
export interface ClickAction {
readonly kind: "click";
readonly target: ComputerPoint;
/** `"left" | "right" | "middle"`. Defaults to `"left"`. */
readonly button?: string;
/** Subset of `["cmd", "ctrl", "alt", "shift"]`. */
readonly modifiers?: readonly string[];
readonly target_hint?: ComputerTargetHint;
}
/**
* Two clicks in rapid succession at `target`.
* @alpha
*/
export interface DoubleClickAction {
readonly kind: "double_click";
readonly target: ComputerPoint;
readonly button?: string;
readonly modifiers?: readonly string[];
readonly target_hint?: ComputerTargetHint;
}
/**
* Move cursor to `target` without clicking.
* @alpha
*/
export interface MouseMoveAction {
readonly kind: "mouse_move";
readonly target: ComputerPoint;
readonly target_hint?: ComputerTargetHint;
}
/**
* Press at `from`, move to `to`, release.
* @alpha
*/
export interface DragAction {
readonly kind: "drag";
readonly from: ComputerPoint;
readonly to: ComputerPoint;
readonly button?: string;
readonly modifiers?: readonly string[];
/** Total drag duration in ms. Implementation may interpolate intermediate points. */
readonly duration_ms?: number;
readonly target_hint?: ComputerTargetHint;
}
/**
* Keyboard text input.
* @alpha
*/
export interface TypeAction {
readonly kind: "type";
readonly text: string;
/**
* Per-character delay in ms. Omitted = implementation default. Set to a
* stable value for deterministic replay of typing cadence.
*/
readonly per_char_delay_ms?: number;
}
/**
* Keyboard combination. Example: `"cmd+c"`, `"ctrl+shift+t"`, `"escape"`.
* @alpha
*/
export interface KeyAction {
readonly kind: "key";
readonly key: string;
}
/**
* Scroll at `target` by `(dx, dy)` wheel deltas.
* @alpha
*/
export interface ScrollAction {
readonly kind: "scroll";
readonly target: ComputerPoint;
readonly dx: number;
readonly dy: number;
}
/**
* Navigate the active browser context to `url`. Cloud-browser-only in
* v1: the headless Playwright runtime in `services/browser-sandbox`
* has no address-bar UI for `key`/`type` to drive, so the spec
* promotion path
* (`spec/computer-use-v1.md` §"No new wire-format actions… real-usage-
* driven, not speculative") fired when the AI hit "navigate to
* tesla.com" against the cloud-browser dispatcher.
*
* Desktop dispatcher (`apps/desktop/src-tauri/src/computer_use.rs`)
* does NOT implement this action — OS-level computer-use has no
* notion of "the active browser context"; the user is in control of
* which app is focused. The dispatcher-parity check at
* `scripts/check-computer-use-dispatcher-parity.ts` carries an
* ALLOWLIST entry naming desktop as deferred until an OS-level
* navigation use-case proves itself.
* @alpha
*/
export interface NavigateAction {
readonly kind: "navigate";
/**
* Target URL. Implementations SHOULD normalize relative-looking
* inputs (`example.com` → `https://example.com`) but MAY reject
* malformed inputs with a `not_supported` failure.
*/
readonly url: string;
}
/**
* element-1 — click on a structurally-addressed element. Server
* resolves `element_id` (issued by a prior `read_page` extraction
* via the stamped `data-motebit-id` attribute), scrolls into view,
* clicks the center of the element's bounding rect.
*
* Prefer this over coordinate-based `click` when the target was
* discovered via `read_page` — durable against viewport, zoom, and
* layout shifts. Coordinate `click` remains for purely-visual tasks
* (clicking on a position seen in pixels).
*
* On staleness (page navigated since read_page, page reloaded,
* element removed by JS) the result fails with `element_not_found`
* and the AI re-reads to refresh the id space.
* @alpha
*/
export interface ClickElementAction {
readonly kind: "click_element";
readonly element_id: string;
}
/**
* element-1 — focus a structurally-addressed element without
* clicking. Useful for setting up `type` calls without the side-
* effects of a click (some fields open dropdowns / dialogs on
* click; focus-only avoids them).
* @alpha
*/
export interface FocusElementAction {
readonly kind: "focus_element";
readonly element_id: string;
}
/**
* element-1 — semantic-intent type. Server focuses the addressed
* element first (so keystrokes can't be swallowed by the page's
* focus state), optionally clears existing value, then types. The
* result envelope carries `text_appeared` truth-feedback so the AI
* doesn't confabulate "I typed it" when the keystrokes went
* nowhere.
*
* Composes the two-step click+type workflow into one semantic
* action — closes the action-truth gap from the witnessed
* 2026-05-08 smoke (AI typed without focus → keystrokes
* swallowed → AI claimed success).
* @alpha
*/
export interface TypeIntoAction {
readonly kind: "type_into";
readonly element_id: string;
readonly text: string;
/** Per-character delay in ms; same shape as the lower-level `type` action. */
readonly per_char_delay_ms?: number;
/**
* If true (default), the field's current value is cleared before
* typing. Mirrors the human "type fresh into this box" intent. Set
* to false to append to existing text.
*/
readonly clear_first?: boolean;
}
/**
* Full action taxonomy. Every concrete action the motebit can request is
* one of these. Exhaustive discriminated union on `kind`.
* @alpha
*/
export type ComputerAction = ScreenshotAction | CursorPositionAction | ClickAction | DoubleClickAction | MouseMoveAction | DragAction | TypeAction | KeyAction | ScrollAction | NavigateAction | ClickElementAction | FocusElementAction | TypeIntoAction;
/**
* Discriminator values — useful for JSON Schema enum and runtime validation.
* @alpha
*/
export declare const COMPUTER_ACTION_KINDS: readonly ["screenshot", "cursor_position", "click", "double_click", "mouse_move", "drag", "type", "key", "scroll", "navigate", "click_element", "focus_element", "type_into"];
/** @alpha */
export type ComputerActionKind = (typeof COMPUTER_ACTION_KINDS)[number];
/**
* One invocation of the `computer` tool. The `action` field holds a nested
* discriminated variant; all action-specific fields live inside that
* object. Impossible states are structurally unrepresentable.
* @alpha
*/
export interface ComputerActionRequest {
/** Open session the action belongs to. */
readonly session_id: string;
readonly action: ComputerAction;
}
/**
* Outcome of a successful observation action's execution. Returned as the
* `data` field of a `ToolResult`. Discriminated by `kind`.
* @alpha
*/
export type ComputerObservationResult = ScreenshotObservation | CursorPositionObservation;
/**
* Structured redaction metadata. Replaces a bare boolean — a verifier can
* now prove *what* was redacted, under *which* policy version, and whether
* the bytes the AI saw are raw or a projection.
* @alpha
*/
export interface ComputerRedaction {
/** `true` iff any region was classified as sensitive and masked. */
readonly applied: boolean;
/**
* Projection shape of the bytes the AI consumed. `"raw"` = unmodified
* capture, `"masked"` = regions replaced with solid fill,
* `"blurred"` = regions convolved, `"cropped"` = sensitive regions
* removed from frame. Surfaces MAY add projection kinds in v1.1.
*/
readonly projection_kind: string;
/** Version string of the sensitivity-classification policy that ran. */
readonly policy_version?: string;
/** Number of regions classified as sensitive in the raw frame. */
readonly classified_regions_count?: number;
/**
* SHA-256 digest of a canonical JSON array of classified regions
* (each region: `{ x, y, w, h, classification }`). Lets a verifier
* replay what was masked without exposing the region list in the
* receipt if policy dictates.
*/
readonly classified_regions_digest?: string;
}
/**
* Screenshot observation. Bytes live in the receipt artifact store keyed
* by `artifact_id` and bound by `artifact_sha256`. When redaction produces
* a distinct projection (e.g. masked image), both the raw and projection
* artifact IDs are referenced so a verifier can fetch either depending on
* authorization.
* @alpha
*/
export interface ScreenshotObservation {
readonly kind: "screenshot";
readonly session_id: string;
/** Artifact ID of the raw capture in the artifact store. */
readonly artifact_id: string;
/** SHA-256 of the raw capture's bytes. */
readonly artifact_sha256: string;
/** `"png" | "jpeg"`. */
readonly image_format: string;
/** Image width in logical pixels. */
readonly width: number;
/** Image height in logical pixels. */
readonly height: number;
/** Unix ms of the capture. */
readonly captured_at: number;
readonly redaction: ComputerRedaction;
/**
* Artifact ID of the redacted projection, when `redaction.applied` and
* the projection differs from the raw capture. When absent, the AI saw
* the raw bytes (redaction.projection_kind === "raw").
*/
readonly projection_artifact_id?: string;
/** SHA-256 of the projection bytes. Paired with `projection_artifact_id`. */
readonly projection_artifact_sha256?: string;
}
/**
* Cursor-position observation — single coordinate pair.
* @alpha
*/
export interface CursorPositionObservation {
readonly kind: "cursor_position";
readonly session_id: string;
readonly x: number;
readonly y: number;
readonly captured_at: number;
}
/**
* A single heading extracted from the page's DOM. `level` is the
* HTML heading level 1-6; `text` is the visible text content. The
* server returns headings in document order so the AI can rebuild
* the page's outline without parsing markup.
* @alpha
*/
export interface ReadPageHeading {
readonly level: number;
readonly text: string;
}
/**
* A single visible link extracted from the page's DOM. `text` is
* the anchor's visible label (innerText); `href` is the absolute
* URL the link points to. Server skips empty-label links and
* `javascript:` / fragment-only hrefs.
* @alpha
*/
export interface ReadPageLink {
readonly text: string;
readonly href: string;
}
/**
* A single typeable input extracted from the page's DOM —
* `` (text-shaped types: text, search, email, url, tel,
* number, password) and `