import type { CuaExecutor, CuaObservation } from "./computer-use.js"; /** * The minimal slice of the real @e2b/desktop Sandbox the executor depends on. A * structural subset of the real class (v2.3.3), so a real Sandbox satisfies it * with no adapter. Methods are typed to return `Promise | void` (and the * screenshot bytes likewise) so a synchronous fake also satisfies the port; the * executor awaits every call, which is correct for both sync and async returns. */ export interface E2BDesktopLike { /** Optional command surface used only for best-effort substrate fallbacks. */ commands?: { run(command: string, options?: { requestTimeoutMs?: number; timeoutMs?: number; }): Promise<{ exitCode?: number; stderr?: string; stdout?: string; }>; }; /** Optional file surface used to transfer typed text without shell-quoting it. */ files?: { write(path: string, data: string | ArrayBuffer, options?: { requestTimeoutMs?: number; useOctetStream?: boolean; }): Promise; }; /** Capture the current desktop frame as PNG bytes (default/'bytes' overload). */ screenshot(): Promise | Uint8Array | Buffer; /** Left click, optionally moving to (x, y) first. */ leftClick(x?: number, y?: number): Promise | void; /** Right click, optionally moving to (x, y) first. */ rightClick(x?: number, y?: number): Promise | void; /** Middle click, optionally moving to (x, y) first. */ middleClick(x?: number, y?: number): Promise | void; /** Double left click, optionally moving to (x, y) first. */ doubleClick(x?: number, y?: number): Promise | void; /** Move the mouse to the given coordinates. */ moveMouse(x: number, y: number): Promise | void; /** Optional fresh pointer read; older/custom desktops retain coordinate-bearing clicks. */ getCursorPosition?(): Promise<{ x: number; y: number; }> | { x: number; y: number; }; /** Scroll the mouse wheel vertically by amount ticks in a direction. */ scroll(direction?: "up" | "down", amount?: number): Promise | void; /** Write text at the current cursor position (the SDK's typing method). */ write(text: string): Promise | void; /** Press a key or chord (the SDK's key method); accepts the keys array. */ press(key: string | string[]): Promise | void; /** Drag from one coordinate tuple to another. */ drag(from: [number, number], to: [number, number]): Promise | void; /** Wait for the given number of milliseconds. */ wait(ms: number): Promise | void; } export interface E2BDesktopExecutorOptions { /** Fallback wait when a wait action carries no ms. Default 500. */ defaultWaitMs?: number; /** * Pixels of CuaAction scroll dy per one SDK scroll tick. The executor maps * abs(dy) / scrollAmountPerTick to the SDK's integer `amount` (floored at 1 for * any nonzero scroll). Default 100. */ scrollAmountPerTick?: number; /** * Optional runtime-only browser state probe. Used for deterministic stopWhen guards. The loop * never persists raw URL/title/text; it only uses them in memory to decide whether to stop. */ observeBrowserState?: () => Promise>; } /** * The stage of the type -> clipboard-paste fallback chain that failed. Public-safe * (a path label, never typed text). Surfaced so a run bundle can tell app focus * from a missing clipboard utility from a failed paste keypress. */ export type CuaTypeFallbackPhase = "clipboard-unavailable" | "clipboard-tempfile" | "clipboard-utility-missing" | "clipboard-command" | "paste-keypress"; /** * A `type` action that failed after both the primary write and the clipboard * paste fallback. Carries a redacted attempt chain (path labels only, never the * typed text), the failing phase, and a sanitized stderr/stdout tail when the * substrate produced one. The loop records `.name` + `.message` into the actor * trace notice, so the bundle proves WHERE the type stopped, not just that it did. */ export declare class CuaTypeFallbackError extends Error { readonly phase: CuaTypeFallbackPhase; readonly attemptChain: readonly string[]; readonly stderrTail?: string; constructor(phase: CuaTypeFallbackPhase, attemptChain: readonly string[], stderrTail?: string, cause?: unknown); } /** * Create a CuaExecutor backed by an E2B desktop (or any structural E2BDesktopLike, * e.g. a CI fake). observe() captures a frame and computes its perceptual * signature; execute() dispatches one CuaAction to the matching desktop method. * Every desktop call is awaited so sync and async implementations both work. No * screenshot or action is ever logged. */ export declare function createE2BDesktopExecutor(desktop: E2BDesktopLike, options?: E2BDesktopExecutorOptions): CuaExecutor; /** * A perceptual hash of a PNG frame for no-progress detection. Decodes the PNG, area-averages it to * a SIGNATURE_GRID x SIGNATURE_GRID grayscale grid, CONTRAST-NORMALIZES that grid, quantizes each * cell to SIGNATURE_LEVELS, and packs the cells into a hex string. Deterministic (no Date, no * random). Returns SIGNATURE_FALLBACK on any decode failure so two unreadable frames compare equal. * * Why normalization, and why this grid (#383). The original was 16x16 at 2 bits per cell with no * normalization. On a 1440x950 desktop that is one cell per ~90x59 px, and on a light-themed web app * the area-average of nearly every cell lands at the top of the range: a measured run had 93% of the * 256 cells pinned to level 3 and levels 0 and 1 never used at all. The hash was effectively a * constant, so renaming a row in a sidebar, adding a list item, or opening a small panel could not * move it — 22 frames across one run produced 5 distinct values, and 9 visibly different consecutive * frames produced ONE. The no-progress backstop read that as a stuck agent and ended a lane that was * a foreign key away from finishing its mission. * * Stretching each frame's own min..max across the full range before quantizing is what makes a * mostly-white UI use the levels it actually has, and the finer grid shrinks a cell to ~45x30 px so * ordinary widget-sized changes survive the averaging. * * This is still a coarse whole-frame hash and it is still only ONE input to the backstop — see the * corroboration rule in computer-use.ts, which is what keeps a blind frame from ending a run on its * own. */ export declare function perceptualSignature(pngBytes: Buffer | Uint8Array): string;