import type { GeometryRect } from "./geometry.js"; import type { ToolExternalEffectIntent } from "../types.js"; export type UiVerb = "invoke" | "focus" | "set_value" | "set_checked" | "select" | "scroll" | "key_chord"; export interface UiNode { id: string; generation: number; role: string; name: string; enabled: boolean; visible: boolean; operations: UiVerb[]; value?: string; checked?: boolean; selected?: string[]; sensitive?: boolean; focused?: boolean; bounds?: GeometryRect; scopeId?: string; scroll?: { x: number; y: number; maxX: number; maxY: number; }; } export interface UiSnapshot { schema: "omnius.ui-state.v1"; sessionId: string; revision: string; capturedAt: number; scope: { kind: "browser" | "native"; id: string; url?: string; app?: string; windowId?: string; }; nodes: UiNode[]; text?: string; /** Per-document browser text; iframe content must not inherit the top-level URL. */ documents?: Array<{ scopeId: string; url: string; mainFrame: boolean; text: string; truncated: boolean; }>; /** Bounded UI navigation context, never a complete page/source read. */ observationCoverage?: { scope: "bounded_ui_observation"; completePageRead: false; availableFrames: number; returnedFrames: number; frameLimit: number; framesTruncated: boolean; /** Normalized original text from returned frames only, including separators. */ aggregateTextScope: "normalized_returned_frames"; aggregateTextOriginalChars: number; aggregateTextReturnedChars: number; aggregateTextLimitChars: number; aggregateTextTruncated: boolean; }; browser?: { engine: string; version: string; protocol: string; capabilities: string[]; }; native?: { platform: "linux" | "win32"; provider: "atspi2" | "uia"; capabilities: string[]; activeWindowId?: string; /** Host accessibility metadata for exact native/OS visual binding; windowId stays opaque. */ activeProcessId?: number; activeWindowBounds?: GeometryRect; }; } export type UiPredicate = { kind: "value"; nodeId: string; value: string; } | { kind: "checked"; nodeId: string; checked: boolean; } | { kind: "focused"; nodeId: string; } | { kind: "selected"; nodeId: string; values: string[]; } | { kind: "visible"; role?: string; name?: string; visible: boolean; } | { kind: "text"; text: string; } | { kind: "url"; url: string; } | { kind: "download_saved"; filename?: string; }; export interface UiArgs { value?: string; checked?: boolean; option?: string; direction?: "up" | "down" | "left" | "right"; keys?: string[]; } export interface UiStep { id: string; verb: UiVerb; target: { nodeId?: string; role?: string; name?: string; scopeId?: string; }; args?: UiArgs; expect: UiPredicate[]; } export interface UiAction { id: string; kind: "action" | "observe" | "abstain"; scopeId: string; revision: string; nodeId?: string; generation?: number; verb?: UiVerb; args?: UiArgs; expect: UiPredicate[]; idempotencyKey: string; description: string; } export interface UiDispatchReceipt { status: "dispatched" | "rejected"; reason?: string; download?: { path: string; sizeBytes: number; sha256: string; verified: boolean; }; details?: unknown; } /** A provider could not complete this read. Never evidence of a missing target. */ export declare class UiObservationUnavailableError extends Error { } export interface UiAdapter { /** Native provider identity; wrappers preserve it without inventing a binding. */ getDesktopVerificationScope?(): import("./desktop-verification.js").DesktopVerificationScope | undefined; prepareStep?(step: UiStep, signal?: AbortSignal): void | Promise; observe(signal?: AbortSignal): Promise; perform(action: UiAction, signal?: AbortSignal): Promise; } export interface UiDecision { choice: string; probabilities?: Record; probabilityKind: "native" | "surrogate" | "none"; provider: string; latencyMs: number; reason?: string; } /** Trusted task intent accompanies the current finite candidate set; it grants no authority. */ export interface UiDecisionContext { goal: string; step: UiStep; } export interface DecisionProvider { decide(state: UiSnapshot, actions: readonly UiAction[], signal?: AbortSignal, context?: UiDecisionContext): Promise; } export interface UiOutcome { status: "verified" | "unresolved" | "rejected" | "abstained" | "cancelled"; action?: UiAction; before?: UiSnapshot; after?: UiSnapshot; receipt?: UiDispatchReceipt; reason?: string; verification?: { timeoutMs: number; intervalMs: number; maxObservations: number; observations: number; elapsedMs: number; }; } /** Host execution boundary shared by a local controller and an admitted runner bridge. */ export interface UiExecutionPort { observe(signal?: AbortSignal): Promise; observeForStep(step: UiStep, signal?: AbortSignal): Promise; enumerate(step: UiStep, state?: UiSnapshot): UiAction[]; execute(id: string, signal?: AbortSignal): Promise; runStep(step: UiStep, provider?: DecisionProvider, signal?: AbortSignal): Promise; } /** Canonical serialization binds every argument/effect, irrespective of object insertion order. */ export declare function uiDigest(value: unknown): string; export declare function uiEffectIntent(action: UiAction): ToolExternalEffectIntent | null; export declare function parseUiStep(input: unknown): UiStep; export declare const UI_STEP_SCHEMA: { readonly type: "object"; readonly additionalProperties: false; readonly required: readonly [ "id", "verb", "target", "expect" ]; readonly properties: { readonly id: { readonly type: "string"; }; readonly verb: { readonly type: "string"; readonly enum: UiVerb[]; }; readonly target: { readonly type: "object"; readonly additionalProperties: false; readonly properties: { [k: string]: { minLength?: number | undefined; type: string; }; }; }; readonly args: { readonly type: "object"; readonly additionalProperties: false; readonly properties: { readonly value: { readonly type: "string"; }; readonly checked: { readonly type: "boolean"; }; readonly option: { readonly type: "string"; }; readonly direction: { readonly type: "string"; readonly enum: readonly [ "up", "down", "left", "right" ]; }; readonly keys: { readonly type: "array"; readonly items: { readonly type: "string"; }; readonly maxItems: 4; }; }; }; readonly expect: { readonly type: "array"; readonly minItems: 1; readonly maxItems: 20; readonly items: { readonly type: "object"; readonly properties: { readonly kind: { readonly type: "string"; readonly enum: readonly [ "value", "checked", "selected", "focused", "visible", "text", "url", "download_saved" ]; }; readonly nodeId: { readonly type: "string"; }; readonly value: { readonly type: "string"; }; readonly checked: { readonly type: "boolean"; }; readonly values: { readonly type: "array"; readonly items: { readonly type: "string"; }; }; readonly role: { readonly type: "string"; }; readonly name: { readonly type: "string"; }; readonly visible: { readonly type: "boolean"; }; readonly text: { readonly type: "string"; }; readonly url: { readonly type: "string"; }; readonly filename: { readonly type: "string"; }; }; }; }; }; };