/** * Capture Agent — IR Types * * All types for the compiled execution model: * preset (natural language) -> ExecutionProgram (typed IR) -> deterministic runtime */ import type { AKTree, BrowserStorageState, BrowserSessionStorageState, OutscaleConfig, VideoCursorTheme, VideoPageSignals } from './types.js'; import type { DeviceConfig, MockupOptions } from './mockup.js'; export type { DeviceConfig }; /** Sentinel value that resolves to the current variant's locale or theme at runtime */ export declare const VARIANT_PLACEHOLDER: "$variant"; export declare const OPCODE_KINDS: readonly ["NAVIGATE", "DISMISS_OVERLAYS", "ASSERT_ROUTE", "ASSERT_SURFACE", "CLICK", "TYPE", "PRESS_KEY", "WAIT_FOR", "SLEEP", "SET_LOCALE", "SET_THEME", "SCROLL", "CAPTURE_SCREENSHOT", "BEGIN_CLIP", "END_CLIP", "HOVER", "SELECT_OPTION", "CHECK", "DOUBLE_CLICK", "DRAG", "CLONE_ELEMENT", "INJECT_MOCK_DATA", "REMOVE_ELEMENT", "SET_ATTRIBUTE"]; export type OpcodeKind = (typeof OPCODE_KINDS)[number]; /** * Soft opcodes are non-blocking — if their action or postcondition fails at * runtime, the runner returns `status: 'skipped'` instead of aborting the * variant. The circuit breaker is not ticked. Used by mock data injection so * a missing template selector logs a warning and the capture proceeds. */ export declare const SOFT_OPCODE_KINDS: ReadonlySet; export declare function isSoftOpcodeKind(kind: OpcodeKind): boolean; /** A single fillable field within a mock data group's templated item. */ export interface MockDataSlot { /** Stable identifier referenced by INJECT_MOCK_DATA.slotMappings[].slot */ name: string; /** Human-readable description shown in the Phase 3 editing UI */ description: string; /** * Optional hint about how the Phase 3 editing UI should render the input. * Canonical values recognized by the dashboard editor are `text`, * `number`, `image`, `color`, `date`, `url`. Any other string is allowed * and the editor falls back to a plain text input. */ hint?: string; } /** One concrete row of values, keyed by slot name. */ export type MockDataRow = Record; /** A mock data group describes one fillable area of the UI (e.g. a table body). */ export interface MockDataGroup { /** Stable identifier referenced by INJECT_MOCK_DATA.groupName. Kebab-case. */ name: string; /** Human-readable description for the editing UI */ description: string; /** Schema of each item in this group */ slots: MockDataSlot[]; /** Default seed values — one row per item the user wants rendered */ defaultValues: MockDataRow[]; /** * If true, after cloning the runtime removes every existing child of the * container that is not a clone. Used to wipe placeholders / real items so * the mocked dataset is the sole content. Default: false. */ replaceExisting?: boolean; } export type PostconditionType = 'route_matches' | 'element_visible' | 'element_absent' | 'text_contains' | 'overlay_dismissed' | 'screenshot_stable' | 'any_change' | 'always'; export interface PostconditionSpec { type: PostconditionType; /** URL pattern (glob) for route_matches */ pattern?: string; /** CSS/AKTree selector for element_visible, element_absent, text_contains */ selector?: string; /** Expected text substring for text_contains */ text?: string; /** Pixel diff threshold (0-1) for screenshot_stable. Default: 0.01 */ threshold?: number; /** Max wait time (ms) for the postcondition to become true. Default: 5000 */ waitMs?: number; } /** * Describes an element semantically, without requiring a CSS selector. * The runtime resolves the target using Playwright semantic locators * (getByRole, getByText, getByLabel) then falls back to AKTree fuzzy matching. * * This is the primary targeting mechanism for programs generated without * modifying the captured app's source code: the descriptor is grounded in * what is visible on screen (role, accessible name, text) rather than in a * pre-planted automation attribute. * * At least one of text/role/label (or a locale-keyed variant) should be * provided. For multilingual apps, the `*ByLocale` maps carry the per-locale * strings extracted from the app's i18n catalogs; the runtime picks the value * matching the active variant locale (exact tag, then primary subtag) and * falls back to the base field. */ export interface SemanticTarget { /** Visible text content of the element (exact or partial match) */ text?: string; /** Locale-keyed visible text (BCP-47-ish keys: "en", "fr", "fr-FR") */ textByLocale?: Record; /** ARIA role: "button", "link", "textbox", "checkbox", "tab", "menuitem", etc. */ role?: string; /** Accessible name: aria-label, associated label text, or title attribute */ label?: string; /** Locale-keyed accessible name */ labelByLocale?: Record; /** Nearby text or heading for disambiguation (e.g. "in the pricing section") */ near?: string; /** Locale-keyed nearby text */ nearByLocale?: Record; /** Placeholder text (for inputs) */ placeholder?: string; /** Locale-keyed placeholder text */ placeholderByLocale?: Record; /** Whether the text match should be exact. Default: false (substring match) */ exact?: boolean; } export interface RecoveryPolicy { /** Number of deterministic retries before escalating. Default: 2 */ retries: number; /** Try selector memory alternatives. Default: true */ useSelectorMemory: boolean; /** Try alternative interaction methods (keyboard, JS, coords). Default: true */ useAltInteraction: boolean; /** Reload the page and retry from last checkpoint. Default: false */ allowReload: boolean; /** Allow LLM healer as last resort. Default: false */ allowHealer: boolean; } export declare const DEFAULT_RECOVERY_POLICY: RecoveryPolicy; interface OpcodeBase { kind: OpcodeKind; /** Human-readable description of what this opcode does */ description: string; postcondition: PostconditionSpec; recovery: RecoveryPolicy; /** Max time (ms) for this opcode to complete, including recovery. Default: 15000 */ timeoutMs: number; /** Max recovery attempts before this opcode is considered failed. Default: 3 */ maxFailures: number; /** * Stable identifier used to anchor narration overlays in `mediaMode='video'` * runs. Kebab-case, 1-64 chars. Optional — only required for opcodes whose * narration segment must survive program rewrites (e.g. WAIT_FOR insertions * before run-time speech generation). Ignored in `screenshot` and `clip` modes. */ stepId?: string; } export interface NavigateOpcode extends OpcodeBase { kind: 'NAVIGATE'; url: string; } export interface DismissOverlaysOpcode extends OpcodeBase { kind: 'DISMISS_OVERLAYS'; } export interface AssertRouteOpcode extends OpcodeBase { kind: 'ASSERT_ROUTE'; /** URL pattern (glob or regex) to assert */ urlPattern: string; } export interface AssertSurfaceOpcode extends OpcodeBase { kind: 'ASSERT_SURFACE'; /** Selector(s) that must be visible */ selectors: string[]; /** If true, ALL selectors must match. If false, ANY one suffices. Default: true */ matchAll: boolean; } export interface ClickOpcode extends OpcodeBase { kind: 'CLICK'; /** * CSS selector built from attributes that already exist in the app * (id, data-testid, aria-label, href …). Optional — when absent, `target` * drives resolution. At least one of `selector`/`target` is required. */ selector?: string; /** Semantic on-screen target — primary when `selector` is absent, fallback when it fails */ target?: SemanticTarget; /** Mouse button: 'right' for context menus, 'middle' for new tab. Default: left click */ button?: 'right' | 'middle'; /** AKTree fingerprint for fuzzy matching fallback */ fingerprint?: string; /** Alternative selectors to try in order */ selectorAlternates?: string[]; } export interface TypeOpcode extends OpcodeBase { kind: 'TYPE'; /** CSS selector from existing attributes. Optional when `target` is provided. */ selector?: string; /** Semantic on-screen target — primary when `selector` is absent, fallback when it fails */ target?: SemanticTarget; text: string; /** Locale-keyed text overrides. When the current variant has a matching locale key, * that value is used instead of `text`. Supports credential placeholders. */ textByLocale?: Record; /** Clear existing value before typing. Default: true */ clearFirst: boolean; fingerprint?: string; selectorAlternates?: string[]; } export interface PressKeyOpcode extends OpcodeBase { kind: 'PRESS_KEY'; key: string; } export interface WaitForOpcode extends OpcodeBase { kind: 'WAIT_FOR'; /** CSS selector to wait for — required unless target is provided */ selector?: string; /** Semantic target fallback — used when selector is not available */ target?: SemanticTarget; /** 'visible' = element visible in viewport, 'attached' = exists in DOM. Default: 'visible' */ state: 'visible' | 'attached'; } /** * Pause execution for a fixed duration. Used by the video run-time TTS step to * align visible actions with the narration audio window. Postcondition is * implicitly `always` — the runtime sleeps then advances. */ export interface SleepOpcode extends OpcodeBase { kind: 'SLEEP'; /** Sleep duration in milliseconds. Capped at 60_000 by the schema. */ durationMs: number; /** * Spoken narration to play during this pause (AUT-57 anchor). Authored by * the IDE assistant — it has full project context (real product copy, * design system, etc.) and writes natural conversational lines. The * run-time TTS step renders the text and rewrites `durationMs` to match the * resulting audio length. Required when `stepId` is set; ignored when * `stepId` is absent (decorative pause without narration). */ narrationText?: string; /** * Locale-keyed spoken narration overrides for multivariant demo videos. * Keys are BCP-47-ish locale tags such as `en`, `fr`, `fr-FR`. At prepare * time AutoKap first tries the exact current locale, then its primary * language subtag. `narrationText` remains the single-locale legacy fallback. */ narrationTextByLocale?: Record; } export interface SetLocaleOpcode extends OpcodeBase { kind: 'SET_LOCALE'; /** Target BCP-47 locale (e.g. "fr", "en-US") */ locale: string; /** Strategy to set locale */ method: 'browser_context' | 'ui_interaction' | 'storage'; /** Selector to interact with (for ui_interaction) */ selector?: string; /** Storage key/value pairs to set (for storage method) */ storageHints?: { storage: 'localStorage' | 'sessionStorage' | 'cookie'; key: string; value: string; }[]; } export interface SetThemeOpcode extends OpcodeBase { kind: 'SET_THEME'; theme: 'light' | 'dark' | '$variant'; method: 'color_scheme' | 'ui_interaction' | 'storage'; selector?: string; storageHints?: { storage: 'localStorage' | 'sessionStorage' | 'cookie'; key: string; value: string; }[]; } export interface ScrollOpcode extends OpcodeBase { kind: 'SCROLL'; direction: 'up' | 'down' | 'left' | 'right'; /** Pixels to scroll. Default: viewport height */ amount?: number; /** Optional target selector to scroll into view */ targetSelector?: string; /** Semantic target to scroll into view (used when targetSelector is absent) */ target?: SemanticTarget; } export interface CaptureScreenshotOpcode extends OpcodeBase { kind: 'CAPTURE_SCREENSHOT'; /** Stable preset capture identifier (page id / element name) for Studio and dev links */ captureId?: string; /** Human-readable preset capture name */ captureName?: string; /** Optional element selector for element-level capture */ elementSelector?: string; /** Optional padding around the captured element. Only applied when `elementSelector` is set. */ outscale?: OutscaleConfig; } export interface BeginClipOpcode extends OpcodeBase { kind: 'BEGIN_CLIP'; /** Stable preset clip identifier for Studio and dev links */ clipId?: string; /** Human-readable preset clip name */ clipName?: string; } export interface EndClipOpcode extends OpcodeBase { kind: 'END_CLIP'; /** Stable preset clip identifier for Studio and dev links */ clipId?: string; /** Human-readable preset clip name */ clipName?: string; } export interface HoverOpcode extends OpcodeBase { kind: 'HOVER'; /** Optional when `target` is provided */ selector?: string; target?: SemanticTarget; fingerprint?: string; selectorAlternates?: string[]; } export interface SelectOptionOpcode extends OpcodeBase { kind: 'SELECT_OPTION'; /** Optional when `target` is provided */ selector?: string; target?: SemanticTarget; /** Select by visible label text */ optionLabel?: string; /** Select by option value attribute */ optionValue?: string; /** Select by zero-based index */ optionIndex?: number; fingerprint?: string; selectorAlternates?: string[]; } export interface CheckOpcode extends OpcodeBase { kind: 'CHECK'; /** Optional when `target` is provided */ selector?: string; target?: SemanticTarget; /** Desired state: true = checked, false = unchecked */ checked: boolean; fingerprint?: string; selectorAlternates?: string[]; } export interface DoubleClickOpcode extends OpcodeBase { kind: 'DOUBLE_CLICK'; /** Optional when `target` is provided */ selector?: string; target?: SemanticTarget; fingerprint?: string; selectorAlternates?: string[]; } /** * Drag an element from point A to point B with an animated cursor. * During clip recordings, the cursor overlay glides from source to * destination along a Bezier curve with press/release visual feedback. * * Destination is either another element (`toSelector` / `toTarget`) or a * relative `offset` from the source center — use the offset form for * sliders, canvas drawing, or any drag whose end point isn't a DOM node. */ export interface DragOpcode extends OpcodeBase { kind: 'DRAG'; /** CSS selector of the source element (the one being dragged). Optional when `target` is provided. */ selector?: string; /** Semantic target for the source element — primary when `selector` is absent */ target?: SemanticTarget; fingerprint?: string; selectorAlternates?: string[]; /** CSS selector of the drop target element (mutually exclusive with `offset`) */ toSelector?: string; /** Semantic fallback for the drop target */ toTarget?: SemanticTarget; /** Alternative destination selectors tried in order */ toSelectorAlternates?: string[]; /** Absolute pixel offset from the source center. Use when the drop point isn't a DOM element (sliders, canvas). */ offset?: { dx: number; dy: number; }; } /** * Duplicate a template element N times into a container. **Soft / non-blocking.** * If the source or container selector is missing, the opcode is skipped and * the variant continues without aborting. */ export interface CloneElementOpcode extends OpcodeBase { kind: 'CLONE_ELEMENT'; /** CSS selector of the element to clone */ sourceSelector: string; /** CSS selector of the container that receives the clones */ containerSelector: string; /** Number of clones to produce (1-500) */ count: number; /** If true, the source template is removed after cloning. Default: false */ removeSource?: boolean; } /** * Orchestrator opcode that fills a fillable area with mock data. Carries * fields for **both** delivery mechanisms simultaneously, and the runtime * applies whichever the opcode declares — typically both: * * - **Clone** (DOM cloning): looks up the group, clones a template element * once per `defaultValues` row, writes each slot value into the cloned * descendants. Provides instant visual feedback even before the app * re-renders. Works for tables, lists, card grids — anything whose children * are pure markup the runtime can mutate. * * - **Trigger** (hidden button + input): writes the JSON-encoded * `defaultValues` into a hidden input the user's app exposed, then clicks a * hidden trigger button. The user's `onClick` handler reads the input value * and re-renders its own component with the seed data. Survives React * re-renders that would otherwise clobber the cloned DOM, and works for * programmatic widgets (Chart.js, Recharts, D3, maps) where the runtime * can't mutate the DOM directly. * * The two mechanisms are complementary, not mutually exclusive: an opcode * should typically declare BOTH so the runtime applies clone first (instant * paint) and then trigger (state-driven, survives re-renders). At least one * mechanism's required fields must be present. * * **Soft / non-blocking** — if neither mechanism produces a successful * application, the opcode is skipped and the variant continues with * `mockDataGroupResults[groupName] = 'skipped'`. If at least one succeeds, * the group is recorded as `'applied'`. */ export interface InjectMockDataOpcode extends OpcodeBase { kind: 'INJECT_MOCK_DATA'; /** Name of the MockDataGroup in ExecutionProgram.mockDataGroups */ groupName: string; /** CSS selector of the container that holds the cloned items */ containerSelector?: string; /** CSS selector of the template element. Default: first child of container. */ templateSelector?: string; /** Override the row count. If absent, uses defaultValues.length. */ count?: number; /** If true, the template element is removed after cloning (empty-state replacement) */ removeTemplate?: boolean; /** Maps each slot name to a target inside a cloned item */ slotMappings?: Array<{ /** Slot name from the MockDataGroup */ slot: string; /** CSS selector RELATIVE to the cloned item */ selector: string; /** If present, set this attribute. If absent, set textContent. */ attribute?: string; }>; /** * CSS selector of the hidden input that receives the JSON-encoded * `defaultValues` array. The user's app reads this value when the trigger * fires. Use a unique `data-ak-fill-input=""` attribute. */ inputSelector?: string; /** * CSS selector of the hidden button that the runner clicks programmatically * (via `element.click()`, bypassing visibility checks). The button's * `onClick` handler reads the input value and re-renders the widget. Use a * unique `data-ak-fill-trigger=""` attribute. */ triggerSelector?: string; } /** Remove all elements matching a selector. **Soft / non-blocking.** */ export interface RemoveElementOpcode extends OpcodeBase { kind: 'REMOVE_ELEMENT'; /** CSS selector — all matches are removed */ selector: string; } /** Set an attribute on the first matching element. **Soft / non-blocking.** */ export interface SetAttributeOpcode extends OpcodeBase { kind: 'SET_ATTRIBUTE'; /** CSS selector of the target element (first match only) */ selector: string; /** Attribute name (e.g. "src", "href", "data-state") */ attribute: string; /** Attribute value */ value: string; } export type ExecutionOpcode = NavigateOpcode | DismissOverlaysOpcode | AssertRouteOpcode | AssertSurfaceOpcode | ClickOpcode | TypeOpcode | PressKeyOpcode | WaitForOpcode | SleepOpcode | SetLocaleOpcode | SetThemeOpcode | ScrollOpcode | CaptureScreenshotOpcode | BeginClipOpcode | EndClipOpcode | HoverOpcode | SelectOptionOpcode | CheckOpcode | DoubleClickOpcode | DragOpcode | CloneElementOpcode | InjectMockDataOpcode | RemoveElementOpcode | SetAttributeOpcode; export interface VariantSpec { id: string; viewport: { width: number; height: number; }; deviceScaleFactor?: number; locale?: string; theme?: 'light' | 'dark'; /** Stable target identifier from the preset config */ targetId?: string; /** Human-readable target label from the preset config */ targetLabel?: string; /** Device frame label (e.g. "iPhone 15 Pro") for mockup rendering */ deviceFrame?: string; /** Per-variant device/browser frame options persisted from the preset config */ mockupOptions?: MockupOptions; } export interface PreconditionSpec { /** * Auth bootstrap is auto-detected from what is configured on the preset: * - If `cookies` / `storageState` / `sessionStorage` are present, the runtime * injects them into the browser context before any opcode runs. * - If `credentials` (email/password) are present, they are substituted * into `{{email}}` / `{{password}}` placeholders inside the program's * `TYPE` opcodes (the program performs the UI login itself). * - If neither is present, the run is anonymous. * * Both mechanisms can coexist (e.g. seed cookies AND fall back to UI login). * * Note: the AI assistant must HARDCODE the login URL directly in the * `NAVIGATE` opcode. The legacy `{{loginUrl}}` placeholder is deprecated * because the field is optional and substituting an empty string crashes * Playwright with a confusing "Cannot navigate to invalid URL" error. */ /** Credentials reference ID (resolved at runtime from CLI config) */ credentialsId?: string; /** * Plain login credentials (decrypted server-side and bridged into the * program at fetch time). Used to substitute `{{email}}` / `{{password}}` * placeholders in TYPE opcodes. **Do not use `{{loginUrl}}` in new * programs** — it's kept for backwards compat with older presets but * deprecated; the agent should hardcode the login URL in NAVIGATE. */ credentials?: { email?: string; password?: string; /** @deprecated Hardcode the login URL directly in NAVIGATE opcodes instead. */ loginUrl?: string; }; /** Pre-loaded storage state */ storageState?: BrowserStorageState; sessionStorage?: BrowserSessionStorageState; /** Cookies to set before execution */ cookies?: Array<{ name: string; value: string; domain: string; path?: string; }>; /** * Active AutoKap Scenario id (AUT-239). When set and `AUTOKAP_SCENARIO_SECRET` * is configured, the runner injects a signed `__ak_scenario=.` cookie * before navigation, so the client app's cooperative scenario layer reads it * server-side (SSR-safe) and serves the named state's fixtures. Inert if the * secret is absent. */ scenario?: string; } export declare const MEDIA_MODES: readonly ["screenshot", "clip", "video"]; export type MediaMode = (typeof MEDIA_MODES)[number]; export interface ArtifactSpec { mediaMode: MediaMode; /** Output format preferences */ format?: { /** For clips: 'gif' | 'mp4' | 'both'. Default: 'gif' */ clipFormat?: 'gif' | 'mp4' | 'both'; /** For screenshots: 'png' | 'jpeg'. Default: 'png' */ screenshotFormat?: 'png' | 'jpeg'; /** * Physical capture surface used while recording. Required when * `mediaMode='video'` and fixed at the delivery frame (1920×1080). * Legacy 2560×1440 programs are accepted for compatibility and normalized * by the CLI before recording. */ captureResolution?: { width: number; height: number; }; /** * Frame rate used by the capture loop. Defaults to platform-tuned values * for `clip` (8 Linux / 15 elsewhere). For `mediaMode='video'` the runtime * pins this to 30 fps unless explicitly overridden. */ captureFps?: number; /** * Final delivery resolution after compositing. Defaults to 1920×1080 for * `mediaMode='video'`. */ deliveryResolution?: { width: number; height: number; }; }; /** Cursor style for clip recordings. Default: 'minimal'. */ cursorTheme?: VideoCursorTheme; /** Max clip duration in seconds. Clips are trimmed if they exceed this. Default: 8. Ignored when `mediaMode='video'`. */ maxClipDurationSec?: number; /** * @deprecated Device mockups are gated SOLELY by a variant's `deviceFrame` (user-defined, * deterministic). The render paths derive mockup application from `deviceFrame` and ignore this * flag as a gate. Kept optional for back-compat with stored programs; do not author it. */ applyMockup?: boolean; /** * @deprecated Per-variant `mockupOptions.showStatusBar` drives the status bar now. Retained only * as a fallback for legacy programs that lack per-variant `mockupOptions`. Do not author it. */ applyStatusBar?: boolean; } export interface ExecutionProgram { presetId: string; /** * Content-revision counter, bumped ONLY by the healer after a selector repair * (program-patcher.ts). Orthogonal to `programSchemaVersion` — it tracks * content changes, never the form of the program. */ programVersion: number; /** * FORM version of the program, driving migrate-on-read (program-migrations.ts). * Optional on the type so hand-built literals (fixtures) stay terse; guaranteed * present at runtime by `upgradeProgram` (run inside parseProgram), which stamps * it to the current value. Absent = v0 (oldest form). Read with * `?? CURRENT_PROGRAM_SCHEMA_VERSION` when stamping. */ programSchemaVersion?: number; /** * Provenance stamp: the engine semantics version the generator compiled this * program against. Informational only — the runtime engine always applies the * current semantics regardless. Absent = legacy (pre-versioning) program. */ engineVersion?: number; mediaMode: MediaMode; baseUrl: string; /** Server-resolved concurrency cap for this run, derived from the owner's plan. */ maxParallelCaptures?: number; variants: VariantSpec[]; preconditions: PreconditionSpec; steps: ExecutionOpcode[]; artifactPlan: ArtifactSpec; /** Global output scale (device pixel ratio). Applied to all variants that don't set their own deviceScaleFactor. */ outputScale?: number; /** Hash of the source preset content, used to detect recompilation needs */ compileFingerprint: string; /** Canonical fingerprint of langs+themes at compilation (e.g. "langs:en,fr|themes:dark,light"). Used for staleness detection. */ variantFingerprint?: string; /** ISO timestamp of compilation */ compiledAt: string; /** Model used for compilation */ compiledWith?: string; /** * Mock data groups available to INJECT_MOCK_DATA opcodes. * Compiled from PresetConfig.mockDataInjection.groups by the server. */ mockDataGroups?: MockDataGroup[]; /** * Server-embedded device frame configs for every `deviceFrame` referenced by * `variants[]`. The CLI seeds its mockup engine with these so it does not * need direct Supabase access (end-users do not have the service role key). * Keyed by the exact deviceFrame string used in the variants. */ deviceConfigs?: Record; /** * Project-level decorative URL used to decorate browser mockups. The CLI * substitutes the captured origin (typically a local dev server) with this * value via `transformBrowserUrl` before baking it into the browser bar. * AUT-269: derived automatically from the project's prod environment base * URL (absent when no prod environment is configured), not a separate field. */ publicUrl?: string; /** * Auth headers attached to the resolved project environment (Bearer token, * Vercel protection bypass, x-api-key, etc.). Injected into the Playwright * BrowserContext so requests to protected staging/preview URLs go through. * Decrypted server-side from `project_environments.auth_headers_encrypted` * and embedded in the signed program envelope. */ environmentHttpHeaders?: Record; /** * Per-project opt-out for third-party analytics blocking (AUT-234). Default * behavior (field absent / `true`) blocks analytics beacons during capture so * a run never registers a phantom "visit". Set to `false` only when the * project disabled it (`projects.block_analytics_enabled = false`). Server-set * BEFORE signing, so it lives inside the signed envelope. */ blockAnalytics?: boolean; } export interface CircuitBreakerConfig { /** Max recovery attempts per opcode. Default: 3 */ maxPerOpcode: number; /** Max total failures per page navigation. Default: 5 */ maxPerPage: number; /** Max total failures per run. Default: 10 */ maxPerRun: number; } export declare const DEFAULT_CIRCUIT_BREAKER: CircuitBreakerConfig; export interface HealerPatch { /** Index of the opcode in the program that was patched */ opcodeIndex: number; /** The original opcode before patching */ originalOpcode: ExecutionOpcode; /** The replacement opcode(s). Usually 1, max 3. */ replacementOpcodes: ExecutionOpcode[]; /** Bounded patch type used by the healer. */ patchType?: 'selector_patch'; /** Optional interaction mode requested by the healer. */ interactionMode?: 'default' | 'keyboard' | 'js_dispatch' | 'coordinates'; /** Why the healer made this change */ reason: string; /** Timestamp */ patchedAt: string; } export type OpcodeResultStatus = 'ok' | 'recovered' | 'skipped' | 'failed'; export interface OpcodeResult { opcodeIndex: number; kind: OpcodeKind; status: OpcodeResultStatus; /** Time taken for this opcode (ms) */ durationMs: number; /** Number of recovery attempts used */ recoveryAttempts: number; /** Recovery strategy that succeeded, if any */ recoveryStrategy?: 'retry' | 'selector_memory' | 'alt_interaction' | 'reload' | 'healer'; /** Error message if failed */ error?: string; } /** * Structured failure category, set on top of the free-text `error`. Lets the * server surface a specific preset state instead of a generic "failed": * `login_failed` = an opcode inside the login window (credential typing → first * post-login assertion) failed, so the credentials are likely wrong. */ export type RunFailureKind = 'login_failed'; export interface VariantResult { variantId: string; success: boolean; opcodeResults: OpcodeResult[]; /** Total duration for this variant (ms) */ durationMs: number; /** Artifact buffers produced */ artifacts: ArtifactResult[]; /** * App version detected on the captured page (meta tag, window global, or * data attribute). Sent to the server with telemetry so freshness tracking * picks up the version live during the capture instead of waiting for the * hourly cron. */ detectedAppVersion?: string | null; error?: string; /** Set when the failure falls inside the login window — see RunFailureKind. */ failureKind?: RunFailureKind; } export interface ArtifactResult { mediaMode: MediaMode; buffer: Buffer; mimeType: string; durationMs?: number; trimStartMs?: number; /** For screenshots: viewport dimensions of the capture */ dimensions?: { width: number; height: number; }; /** Generated alt text for accessibility */ altText?: string; /** Final URL at the time the artifact was produced */ captureUrl?: string; /** Document title at the time the artifact was produced (page.title()) */ pageTitle?: string; /** Step index that produced the artifact */ stepIndex?: number; /** Human-readable label for the artifact */ stepDescription?: string; /** Variant metadata used by persistence and gallery views */ variantId?: string; /** Stable preset capture identifier for screenshots */ captureId?: string; /** Human-readable preset capture name for screenshots */ captureName?: string; /** Stable preset clip identifier for clips */ clipId?: string; /** Human-readable preset clip name for clips */ clipName?: string; captureType?: 'fullpage' | 'element'; elementSelector?: string; /** Favicon extracted from the captured page */ tabIconData?: Buffer; tabIconMimeType?: string; /** * AUT-240 (Layer 4): the capture was produced under a degraded signal — an * AKTree probe that kept throwing was assumed-OK as a last resort, or the page * never reached a visually-stable state. "Assume OK, but flag it." Q4 decision: * produce-only for now (a downstream consumer in gallery / post-capture * verification is a later phase); no LLM is forced off this flag. */ lowConfidence?: boolean; /** Why the artifact was flagged low-confidence (human-readable). */ lowConfidenceReason?: string; /** * AUT-241 — navigation-watcher warnings captured while this clip/video was * recording (e.g. a full document load mid-take = white flash + cursor loss). * Carried up to `RunResult.warnings`; diagnostic only, never fails the run. */ warnings?: string[]; } export type LLMStepType = 'capture_verification' | 'alt_text_generation' | 'healer_invocation'; export interface LLMStepUsage { stepType: LLMStepType; generationId: string | null; model: string; promptTokens: number; completionTokens: number; } export interface RunTelemetry { /** Total LLM calls (compilation excluded) */ llmCallCount: number; /** Total LLM cost in EUR */ llmCostEur: number; /** Detailed per-call LLM usage for cost logging */ llmStepUsages: LLMStepUsage[]; /** Total opcodes executed across all variants */ totalOpcodes: number; /** Opcodes that needed recovery */ recoveredOpcodes: number; /** Opcodes that failed even after recovery */ failedOpcodes: number; /** Soft opcodes that were skipped (e.g. mock data injection failures) */ skippedOpcodes: number; /** Number of healer invocations */ healerInvocations: number; /** Circuit breaker trips */ circuitBreakerTrips: number; /** Per-group mock data application result */ mockDataGroupResults?: Record; } /** * Per-opcode timing entry emitted when `mediaMode='video'`. Consumed by the * compositor and telemetry tooling to align narration overlays and retain * interaction metadata. `bbox` is captured BEFORE the action runs (the * targeted element may move or disappear after). */ export interface OpcodeTiming { stepIndex: number; stepId?: string; opcodeKind: OpcodeKind; variantId: string; /** clipId of the active BEGIN_CLIP context, if any */ clipId?: string; /** Wall-clock ms relative to the start of the active clip recording */ timecodeStartMs: number; timecodeEndMs: number; /** Bounding box of the targeted element at action time (page coords). null if no DOM target. */ bbox?: { x: number; y: number; width: number; height: number; } | null; /** * For TYPE opcodes captured in clipCursor mode: timestamp (ms relative to the * active clip start) of each individual keystroke produced by `humanType`. * Drives keyboard SFX per-keystroke in the video compositor. Empty/undefined * for non-TYPE opcodes and for typing paths that bypass humanType. */ keystrokeOffsetsMs?: number[]; /** * For CLICK / DOUBLE_CLICK / CHECK opcodes captured in clipCursor mode: * timestamp (ms relative to the active clip start) at which Playwright * dispatched each actual click — measured AFTER the cursor animation * settled on the target. Drives mouse SFX in sync with the visible click. * Empty/undefined for opcodes whose adapter doesn't surface the timestamp. */ clickOffsetsMs?: number[]; } export interface RunResult { programId: string; success: boolean; variantResults: VariantResult[]; telemetry: RunTelemetry; /** Healer patches accumulated during the run. Only propagate to server if success=true */ healerPatches: HealerPatch[]; /** Total run duration (ms) */ totalDurationMs: number; /** * Per-opcode timing entries — populated when `mediaMode='video'`. Empty * array for `screenshot` and `clip` modes. */ opcodeTimings: OpcodeTiming[]; /** * First non-null `detectedAppVersion` from the variants. Used by the server * telemetry route to bypass the cron-detected version and stamp the preset * with the version actually captured on the page. */ detectedAppVersion?: string | null; /** * AUT-241 — non-fatal warnings aggregated from every clip/video recording in * the run (full document loads mid-take, unexpected page-side navigations). * Empty/undefined when nothing was flagged. Anti-cut policy surfaces these * instead of masking the cut; deployed presets are grandfathered at run. */ warnings?: string[]; error?: string; /** First non-null variant `failureKind` — see RunFailureKind. */ failureKind?: RunFailureKind; } export interface WaitCondition { selector: string; state: 'visible' | 'attached'; timeoutMs: number; } /** * Cheap, side-effect-free snapshot of page activity (AUT-240, Layer C). * Compared across polls by the runner's progress watchdog to distinguish a * slow-but-progressing page (extend the wait) from a genuinely stuck one (cut). */ export interface ProgressSnapshot { /** * Monotonic count of FIRST-PARTY network lifecycle events * (request/finished/failed) — same site as the live page origin. Third-party * telemetry is excluded so it cannot masquerade as progress. */ networkEventCount: number; /** First-party requests issued but not yet finished or failed. */ inflightRequests: number; /** `Date.now()` of the last observed first-party network lifecycle event. */ lastNetworkActivityAtMs: number; /** `document.readyState`, or 'unknown' if unreadable. */ readyState: string; /** Total element count — a cheap DOM-churn signal — or -1 if unreadable. */ domNodeCount: number; /** True when the readability probe threw (itself a sign of navigation). */ navigating?: boolean; } /** Result of `RuntimeAdapter.waitForVisuallyStable` (AUT-240, Layer B). */ export interface VisualStabilityResult { /** Whether the page reached a clean, stable state before the deadline. */ stable: boolean; /** Human-readable explanation (which signal stayed noisy, if any). */ reason: string; /** Total time spent stabilizing (ms). */ waitedMs: number; } export interface ClickOptions { /** Force click even if element is covered */ force?: boolean; /** Use keyboard (Tab + Enter) instead of mouse */ useKeyboard?: boolean; /** Use JS dispatch instead of Playwright click */ useJsDispatch?: boolean; /** Click by coordinates from bounding box */ coordinates?: { x: number; y: number; }; /** Mouse button. Default: 'left' */ button?: 'left' | 'right' | 'middle'; /** * Fired with `Date.now()` right before Playwright dispatches the actual * click — i.e. AFTER the visible cursor animation has settled on the * target. The runner converts the wall-clock to clip-relative offsets so * the video compositor can fire mouse SFX in lock-step with the visible * click (instead of when the cursor was still travelling). */ onClick?: (timestampMs: number) => void; } export interface ClickByTargetOptions { selector?: string; target?: SemanticTarget; selectorAlternates?: string[]; /** Active variant locale, used to resolve the target's `*ByLocale` maps */ locale?: string; onClick?: (timestampMs: number) => void; } /** Shared options for the semantic `*ByTarget` adapter methods. */ export interface TargetResolveOptions { selector?: string; target?: SemanticTarget; selectorAlternates?: string[]; /** Active variant locale, used to resolve the target's `*ByLocale` maps */ locale?: string; } export interface MouseActionOptions { /** Same semantics as `ClickOptions.onClick` — fires right before the * actual click is dispatched (CHECK / DOUBLE_CLICK). */ onClick?: (timestampMs: number) => void; } export interface RecordingOptions { mediaMode: 'clip' | 'video'; /** * Physical recording surface. Video capture records the delivery surface * directly (1920×1080 by default). */ captureResolution?: { width: number; height: number; }; /** Override the capture loop frame rate. Clamped to 1..30 by the loop. */ captureFps?: number; } export interface RecordingResult { buffer: Buffer; durationMs: number; mimeType: string; trimStartMs?: number; /** * AUT-241 — human-readable warnings collected by the navigation watcher * during this recording window (e.g. a full document load = white flash + * cursor loss). Surfaced up to `RunResult.warnings`; never fails the run. */ warnings?: string[]; } export interface TypeOptions { /** * Called once per keystroke produced by `humanType`, with the absolute * wall-clock timestamp (`Date.now()`) of the keystroke. The runner converts * those to clip-relative offsets stored on `OpcodeTiming.keystrokeOffsetsMs` * so the compositor can fire per-keystroke SFX. Only fires in clipCursor * mode (the only path that produces visible per-key animation). */ onKeystroke?: (timestampMs: number) => void; } export interface RuntimeAdapter { navigate(url: string): Promise; getCurrentUrl(): Promise; getAKTree(): Promise; getPageSignals(): Promise; click(selector: string, options?: ClickOptions): Promise; type(selector: string, text: string, clearFirst?: boolean, opts?: TypeOptions): Promise; pressKey(key: string): Promise; scroll(direction: 'up' | 'down' | 'left' | 'right', amount?: number): Promise; scrollIntoView(selector: string): Promise; waitFor(condition: WaitCondition): Promise; dismissOverlays(): Promise<{ dismissed: boolean; method: string | null; }>; takeScreenshot(): Promise; takeElementScreenshot?(selector: string, outscale?: OutscaleConfig): Promise; takeCleanScreenshot(): Promise; beginRecording(options: RecordingOptions): Promise; endRecording(): Promise; /** * Page-coord bounding box of the first element matching `selector`. Returns * null if the selector does not match a visible element. Used by the * `mediaMode='video'` runtime to capture interaction metadata before each * visible action. Optional — implementations that cannot resolve a bbox * should leave this method off and the runtime will record `bbox: null`. */ getElementBoundingBox?(selector: string): Promise<{ x: number; y: number; width: number; height: number; } | null>; setLocale(locale: string): Promise; setColorScheme(scheme: 'light' | 'dark'): Promise; reloadPage?(): Promise; writeStorageHint?(params: { storage: 'localStorage' | 'sessionStorage' | 'cookie'; key: string; value: string; kind: 'locale' | 'theme'; }): Promise; /** Extract the page's favicon as a buffer. Returns null if unavailable. */ extractFavicon?(): Promise<{ buffer: Buffer; mimeType: string; } | null>; /** * Document title of the current page (Playwright's `page.title()`). Captured * at screenshot time and stored on the artifact metadata so browser mockups * can render the actual page title instead of just the hostname. Optional — * adapters that cannot resolve a title should leave this method off. */ getPageTitle?(): Promise; /** * Text content of the first element matching `selector`, read live from the * DOM via Playwright (`locator().first().textContent()`). Preferred over the * AKTree for `text_contains` postconditions (AUT-240, Layer A). Returns null * if the selector misses or the read fails. Optional — adapters without it * make `text_contains` fall back to the AKTree. */ getTextContent?(selector: string): Promise; /** * Read the captured app's version from the live page (meta tag, window * global, or data attribute). Mirrors `extractAppVersionFromHtml` server-side * and `__AUTOKAP_VERSION__` lookup. Returns null if no marker is present. */ detectAppVersion?(): Promise; close(): Promise; /** Click an element by semantic target. Falls back to selector if target not found. */ clickByTarget?(opts: ClickByTargetOptions): Promise; /** Type into an element by semantic target. */ typeByTarget?(opts: TargetResolveOptions, text: string, clearFirst?: boolean, typeOpts?: TypeOptions): Promise; /** Wait for an element by semantic target. */ waitForTarget?(opts: TargetResolveOptions, timeoutMs?: number): Promise; /** Scroll an element into view by semantic target. */ scrollIntoViewByTarget?(opts: TargetResolveOptions): Promise; /** Select a dropdown option on an element resolved by semantic target. */ selectOptionByTarget?(opts: TargetResolveOptions, option: { label?: string; value?: string; index?: number; }): Promise; /** Check/uncheck an element resolved by semantic target. */ checkByTarget?(opts: TargetResolveOptions, checked: boolean, mouseOpts?: MouseActionOptions): Promise; /** Double-click an element resolved by semantic target. */ doubleClickByTarget?(opts: TargetResolveOptions, mouseOpts?: MouseActionOptions): Promise; hover?(selector: string): Promise; hoverByTarget?(opts: TargetResolveOptions): Promise; selectOption?(selector: string, option: { label?: string; value?: string; index?: number; }): Promise; check?(selector: string, checked: boolean, opts?: MouseActionOptions): Promise; doubleClick?(selector: string, opts?: MouseActionOptions): Promise; /** * Drag the source element from point A to point B with an animated cursor * when a clip is recording. Destination is either another element * (`toSelector` / `toTarget`) or an `offset` from the source center. */ drag?(opts: { selector?: string; target?: SemanticTarget; selectorAlternates?: string[]; toSelector?: string; toTarget?: SemanticTarget; toSelectorAlternates?: string[]; offset?: { dx: number; dy: number; }; /** Active variant locale, used to resolve the targets' `*ByLocale` maps */ locale?: string; }): Promise; /** Clone an element N times into a container. Throws if either selector misses. */ cloneElement?(opts: { sourceSelector: string; containerSelector: string; count: number; removeSource?: boolean; }): Promise<{ clonedCount: number; }>; /** Set an attribute on the first matching element. Throws if the selector misses. */ setAttribute?(opts: { selector: string; attribute: string; value: string; }): Promise; /** Set the textContent of the first matching element. Throws if the selector misses. */ setTextContent?(opts: { selector: string; text: string; }): Promise; /** Remove all matching elements. Throws only if no elements match. */ removeElement?(opts: { selector: string; }): Promise<{ removedCount: number; }>; /** * Set the `value` of an input/textarea/select via the React-aware native * setter, then dispatch `input` and `change` events so React (and other * controlled-component frameworks) detect the change. Throws if the selector * misses. Used by INJECT_MOCK_DATA strategy='trigger'. */ setInputValue?(opts: { selector: string; value: string; }): Promise; /** * Click an element via JavaScript (`element.click()`), bypassing visibility * and enabled checks. Useful for clicking hidden trigger buttons that * `display:none` or `pointer-events:none` would block. Throws if the * selector misses. Used by INJECT_MOCK_DATA strategy='trigger'. */ clickHidden?(opts: { selector: string; }): Promise; /** * Cheap snapshot of page activity used by the runner's progress watchdog to * decide whether a wait is "slow-but-progressing" (keep waiting) or "stuck" * (cut). Must never reject — implementations catch internally and set * `navigating: true` when the readability probe throws. Adapters that cannot * provide it leave it off; the runner then falls back to fixed budgets. */ getProgressSnapshot?(): Promise; /** * Wait for the page to be visually stable before a screenshot (Layer B): * fonts ready, images loaded, no semantic loaders ([aria-busy]/progressbar) * visible, DOM quiet, with a bounded pixel-convergence fallback. Best-effort — * never throws and never blocks the capture: when it cannot reach a clean * state it returns `stable: false` with a reason and the runner captures * anyway. Adapters that cannot provide it leave it off; the runner falls back * to the legacy `smartWaitForStability`. */ waitForVisuallyStable?(options?: { maxWaitMs?: number; }): Promise; }