import type { BrowserContextOptions } from 'playwright'; export type BrowserStorageState = Exclude; export type BrowserSessionStorageState = Record>; export type SessionProfileAuthState = 'authenticated' | 'login_required' | 'unknown'; export type SessionProfileValidationStatus = 'valid' | 'repaired' | 'invalid' | 'unknown'; export type SessionProfileSource = 'run_shared_auth' | 'persisted_variant' | 'live_variant_state' | 'sequential_handoff' | 'none'; export type SessionValidationDiagnosticOutcome = 'match' | 'mismatch' | 'ambiguous'; export type SessionValidationDiagnosticConfidence = 'low' | 'medium' | 'high'; export interface SessionValidationDiagnostic { outcome: SessionValidationDiagnosticOutcome; confidence: SessionValidationDiagnosticConfidence; reasons: string[]; } export interface ValidatedSessionProfile { storageState?: BrowserStorageState; sessionStorage?: BrowserSessionStorageState; authState: SessionProfileAuthState; accountLabel?: string | null; detectedLang?: string | null; detectedTheme?: 'light' | 'dark' | null; validatedStartUrl?: string | null; lastKnownUrl?: string | null; summary?: string | null; validationStatus: SessionProfileValidationStatus; lastUsedAt?: string | null; profileVersion?: number; } export interface AgentRunHint { key: string; message: string; severity: 'high' | 'medium' | 'low'; source: 'agent_error' | 'session_profile'; } export type CapturePageIdentityKind = 'gallery' | 'modal_selection' | 'modal_configuration' | 'editor_route' | 'detail_route' | 'unknown'; export interface CapturePageIdentity { kind: CapturePageIdentityKind; summary: string; subjectTokens: string[]; dialogTarget: boolean; dedicatedRoute: boolean; } export type CaptureExpectedSurfaceKind = "gallery" | "dialog_selection" | "dialog_configuration" | "editor_route" | "detail_route" | "route" | "unknown"; export type CaptureDialogPolicy = "must_be_open" | "must_be_closed" | "allow_either"; export type CaptureRoutePolicy = "exact_or_descendant" | "same_origin" | "ignore"; export type CaptureReusePolicy = "verified_only" | "same_surface_only" | "never"; export type CaptureVariantPolicy = "must_match" | "prefer_match" | "ignore"; export interface CapturePageContract { expectedSurfaceKind: CaptureExpectedSurfaceKind; dialogPolicy: CaptureDialogPolicy; routePolicy: CaptureRoutePolicy; reusePolicy: CaptureReusePolicy; variantPolicy: CaptureVariantPolicy; } export type CaptureSessionTrust = "verified" | "degraded" | "dirty"; export interface CaptureCheckpointEvidence { urlMatched: boolean; dialogMatched: boolean; authMatched: boolean; variantMatched: boolean; strongSurfaceObserved: boolean; surfaceMatched?: boolean; strongSurfaceSignature?: string | null; primarySurface?: string | null; overlaySurface?: string | null; navigationSurface?: string | null; configurationSurface?: string | null; reasons: string[]; } export interface CaptureTransitionDecision { mode: CaptureSessionTrust; contract: CapturePageContract; evidence: CaptureCheckpointEvidence; reasons: string[]; } export interface CaptureBaselineResult extends CaptureTransitionDecision { shouldNavigateToCanonical: boolean; shouldRecreateContext: boolean; allowFastPath: boolean; } export type CaptureSuccessVerdict = 'candidate_match' | 'blocked' | 'terminal_match'; export type VariantVerdict = 'matched' | 'mismatched' | 'ambiguous' | 'not_requested'; export type CaptureStage = 'session_bootstrap' | 'page_reachability' | 'variant_hygiene' | 'capture_readiness' | 'persist_and_handoff'; export type RegressionCause = 'auth_mismatch' | 'route_mismatch' | 'dialog_mismatch' | 'variant_hygiene_pending' | 'surface_too_weak' | 'repair_handoff' | 'stability_pending' | 'duplicate_candidate' | 'unknown'; export interface BlockerSet { stage: CaptureStage; reasons: string[]; } export interface StageGate { stage: CaptureStage; blockers: BlockerSet; regressionCause?: RegressionCause | null; } export interface ObservedTaskState { authState: SessionProfileAuthState | 'ambiguous'; pageIdentity: CapturePageIdentity | null; routeState: 'matched' | 'same_origin' | 'mismatched' | 'unknown'; dialogState: 'open' | 'closed' | 'ambiguous' | 'unknown'; variantState: VariantVerdict; loadingState: 'stable' | 'loading' | 'unstable' | 'unknown'; surfaceConfidence: SessionValidationDiagnosticConfidence; goalSatisfaction: CaptureSuccessVerdict; } export interface VariantValidatedCapture { pageId: string; prompt: string; url: string; assessment: string; fingerprint?: string | null; identity?: CapturePageIdentity | null; } export type VariantCaptureStatus = 'pending' | 'in_progress' | 'completed' | 'blocked'; export type CaptureRunPhase = 'bootstrap_variant' | 'prepare_variant' | 'reach_capture_state' | 'stabilize' | 'capture' | 'checkpoint' | 'recover' | 'complete'; export type CaptureObjective = 'capture' | 'repair'; export type CaptureRepairCause = 'lang' | 'theme' | 'auth' | 'navigation' | 'overlay' | 'readiness'; export type RepairAuthTarget = 'authenticated' | 'login_screen' | 'neutral'; export type CaptureActionOrigin = 'main_plan' | 'repair_subplan' | 'deterministic' | 'replay' | 'preflight'; export interface CaptureCursor { variantId: string; pageId: string; targetId: string; phase: CaptureRunPhase; resumeFromActionIndex: number; lastVerifiedCheckpointId?: string | null; resumeUrl?: string | null; } export interface RepairTicket { id: string; cause: CaptureRepairCause; status: 'pending' | 'in_progress' | 'resolved' | 'failed'; summary: string; cursor: CaptureCursor; expectedState: { lang?: string | null; theme?: 'light' | 'dark' | null; authState?: SessionProfileAuthState; authTarget?: RepairAuthTarget; url?: string | null; pageId?: string | null; pageIdentity?: CapturePageIdentity | null; blockingReason?: string | null; }; retries: { deterministicRemaining: number; memoryRemaining: number; agentRemaining: number; }; attempts: number; createdAt: string; updatedAt: string; } export interface CaptureCheckpoint { id: string; variantId: string; pageId: string; targetId: string; summary: string; url?: string | null; canonicalUrl?: string | null; resumeUrl?: string | null; pageIdentity?: CapturePageIdentity | null; dialogExpected?: boolean; identityKind?: CapturePageIdentityKind | null; surfaceSignature?: string | null; strongSurfaceSignature?: string | null; pageFingerprint?: string | null; sessionTrust?: CaptureSessionTrust | null; checkpointEvidence?: CaptureCheckpointEvidence | null; checkpointType?: 'route' | 'dialog_open' | 'config_open' | 'content_ready' | 'gallery_detail' | 'editor_ready' | 'unknown'; verifiedAt: string; } export interface VariantCaptureRepairRecord { ticketId: string; pageId: string; cause: CaptureRepairCause; status: 'resolved' | 'failed'; summary: string; checkpointId?: string | null; attemptedAt: string; } export interface CaptureTimelineEvent { type: 'phase_transition' | 'repair_started' | 'repair_resolved' | 'repair_failed' | 'memory_hit' | 'memory_stale' | 'checkpoint_saved' | 'capture_completed' | 'capture_blocked'; timestamp: number; variantId: string; pageId?: string; targetId?: string; phase?: CaptureRunPhase; objective?: CaptureObjective; message: string; repairTicketId?: string; cause?: CaptureRepairCause; checkpointId?: string; metadata?: Record; } export interface CaptureRunState { variantId: string; phase: CaptureRunPhase; objective: CaptureObjective; cursor: CaptureCursor; activeRepairTicket?: RepairTicket | null; checkpoints: CaptureCheckpoint[]; timeline: CaptureTimelineEvent[]; } export interface VariantCaptureManifest { expectedPageIds: string[]; currentPageId?: string | null; currentPageIdentity?: CapturePageIdentity | null; promptFingerprint?: string | null; pagePromptFingerprints?: Record; completedPages: string[]; remainingPages: string[]; previousValidatedCaptures: VariantValidatedCapture[]; captureStatuses?: Record; lastCheckpointId?: string | null; blockedReason?: string | null; recoveryAttempts?: Record; repairHistory?: VariantCaptureRepairRecord[]; } export interface CaptureHandoffContext { previousPageId?: string | null; previousPrompt?: string; currentUrl: string; pageTitle?: string | null; authState: SessionProfileAuthState; accountLabel?: string | null; currentLang?: string | null; currentTheme?: 'light' | 'dark' | null; summary: string; selectorHints?: string[]; navigationHints?: string[]; selectorMemory?: Record; trust?: CaptureSessionTrust; strongCheckpointId?: string | null; derivedFromRepair?: boolean; } export interface BrowserOptions { headed: boolean; viewport: { width: number; height: number; }; /** Emulated device pixel ratio (CSS px -> physical px). */ deviceScaleFactor?: number; /** BCP-47 language tag (e.g. "fr", "fr-FR"). Sets browser locale and Accept-Language. */ lang?: string; /** Color scheme for the browser context. */ colorScheme?: 'light' | 'dark'; /** Optional persisted cookies/localStorage captured during preparation. */ storageState?: BrowserStorageState; /** * Extra HTTP headers injected on every navigation. Used to carry the * environment-level auth (Bearer token, Vercel protection bypass, x-api-key, * etc.) configured on the resolved `project_environments` row. * * These headers are sent to ALL requests the BrowserContext makes during * capture, including cross-origin requests (third-party CDN, analytics). * The dashboard surfaces a warning so users only put environment-scoped * secrets here. */ extraHttpHeaders?: Record; /** * When `false`, the engine does NOT block third-party web-analytics beacons * during capture (AUT-234). Default behavior (`undefined` / `true`) blocks * them so a capture never registers a phantom "visit" in the site's * analytics. Opt-out is a per-project setting (`projects.block_analytics_enabled`). */ blockAnalytics?: boolean; } export interface OutscaleConfig { /** Uniform padding on all 4 sides (pixels). */ padding?: number; /** Per-side overrides (pixels). Take priority over padding. */ paddingTop?: number; paddingRight?: number; paddingBottom?: number; paddingLeft?: number; /** Percentage-based padding relative to element dimensions (0–100). */ paddingPercent?: number; /** Clamp the capture zone to document bounds. Default: true. */ clampToViewport?: boolean; /** Background fill color when capture zone exceeds rendered content. Default: transparent. */ backgroundColor?: string; } export interface IsolatedElement { name: string; description: string; /** * Optional source page id. When provided, isolated element capture is performed * from that validated named page instead of the variant's final page. */ sourcePageId?: string; /** @deprecated Use outscale.padding instead. Kept for backward compatibility. */ padding?: number; outscale?: OutscaleConfig; } export type VideoStepType = 'navigate' | 'dismiss_overlays' | 'click' | 'type' | 'select_option' | 'scroll' | 'wait' | 'hover' | 'drag' | 'key' | 'highlight' | 'assert_url' | 'assert_text' | 'assert_element' | 'assert_page'; export type VideoAssertMatchMode = 'equals' | 'contains' | 'regex'; export type VideoAssertElementState = 'visible' | 'attached'; export type VideoRecordingIntent = 'visible' | 'prepare_only'; export type VideoTargetLabelMatchMode = 'exact' | 'contains' | 'token_overlap'; export interface VideoPageExpectation { urlPatterns?: string[]; titlePatterns?: string[]; textPatterns?: string[]; navPatterns?: string[]; breadcrumbPatterns?: string[]; selectors?: string[]; locale?: string; theme?: 'light' | 'dark'; pageIdentity?: CapturePageIdentity | null; minConfidence?: number; } export interface VideoStepTarget { selector?: string; selectorAlternates?: string[]; href?: string; label?: string; labelMatchMode?: VideoTargetLabelMatchMode; tag?: string; role?: string; containerLabel?: string; containerSelector?: string; coordinates?: { x: number; y: number; }; /** * Snapshot-scoped index hint. Only reliable when paired with the originating coherence key. * It must never be treated as a stable persisted selector by itself. */ index?: number; coherenceKey?: string; } export interface VideoObservationSnapshot { coherenceKey?: string; interactiveElements: InteractiveElement[]; pageSignals: VideoPageSignals; pageIdentity?: CapturePageIdentity | null; } export interface VideoVariantOption { label: string; value?: string | null; selected?: boolean; } export interface VideoVariantControl { kind: 'locale' | 'theme' | 'unknown'; mechanism: 'select' | 'button' | 'link' | 'toggle' | 'menuitem' | 'radio' | 'custom'; selector: string; label: string; value?: string | null; href?: string | null; tag: string; role: string; options?: VideoVariantOption[]; } export interface VideoStorageHint { storage: 'localStorage' | 'sessionStorage'; key: string; kind: 'locale' | 'theme' | 'unknown'; valueSample: string; } export interface VideoChromeThemeSample { area: 'body' | 'header' | 'nav' | 'aside' | 'toolbar'; selector: string; background: string | null; color: string | null; luminance: number | null; } export interface VideoAuthHints { hasPasswordField: boolean; hasEmailField: boolean; hasAuthForm: boolean; loginButtons: string[]; logoutButtons: string[]; accountMenuLabels: string[]; accountLikeText: string[]; } export interface VideoPageSignals { url: string; title: string; htmlLang: string | null; canonicalUrl: string | null; hreflangs: string[]; headings: string[]; navLabels: string[]; breadcrumbLabels: string[]; visibleText: string; localeHints: string[]; detectedTheme: 'light' | 'dark' | null; preferredColorScheme: 'light' | 'dark'; themeRootHints?: string[]; chromeThemeSamples?: VideoChromeThemeSample[]; authHints?: VideoAuthHints; variantControls: VideoVariantControl[]; storageHints: VideoStorageHint[]; } export type VideoStepStrategy = 'primary' | 'wait_short' | 'scroll_target' | 'selector_fallback' | 'retry'; export interface VideoStepAttemptTrace { stepId: string; stepIndex: number; attemptIndex: number; strategy: VideoStepStrategy; selector?: string; resolvedTargetSummary?: string; success: boolean; reason?: string; } export interface VideoStepArtifact { runId?: string; variantId?: string; stepId: string; stepIndex: number; attemptIndex: number; phase: 'dry_run' | 'recording'; strategy: VideoStepStrategy; urlBefore: string; urlAfter: string; selectorUsed?: string; resolvedTargetSummary?: string; reason?: string; beforeScreenshot?: Buffer; afterScreenshot?: Buffer; } export interface VideoSelectorMemoryUpdate { stepSignature: string; selector: string; target?: VideoStepTarget; source: 'deterministic' | 'llm_fix' | 'manual'; success: boolean; } export type VideoDebugArtifactsMode = 'off' | 'failed_only' | 'all_dry_run'; export type VideoCursorTheme = 'minimal' | 'macos' | 'windows'; export interface VideoRunOptions { enableMp4?: boolean; burnedMockup?: boolean; debugArtifacts?: VideoDebugArtifactsMode; cursorTheme?: VideoCursorTheme; preflightRequired?: boolean; } export interface VideoStep { id: string; type: VideoStepType; description: string; recordingIntent?: VideoRecordingIntent; url?: string; urlPattern?: string; target?: VideoStepTarget; selector?: string; scopeSelector?: string; coordinates?: { x: number; y: number; }; toTarget?: VideoStepTarget; toSelector?: string; toCoordinates?: { x: number; y: number; }; text?: string; optionLabel?: string; optionValue?: string; optionIndex?: number; matchMode?: VideoAssertMatchMode; state?: VideoAssertElementState; direction?: 'up' | 'down' | 'left' | 'right'; amount?: number; key?: string; /** Optional drag animation duration */ durationMs?: number; /** Timeout for assertion steps */ timeoutMs?: number; /** Multi-signal page verification for robust route/locale/state checks. */ pageExpectation?: VideoPageExpectation; /** Runtime post-condition for actions that must land on a specific page/state. */ expectedPageAfter?: VideoPageExpectation; /** Milliseconds to wait before executing this step */ waitMs?: number; /** Milliseconds to pause after executing this step (for pacing in recording) */ postStepWaitMs?: number; } export interface VideoPlan { title: string; estimatedDurationSec: number; steps: VideoStep[]; startUrl: string; } export type VideoPhase = 'planning' | 'dry_run' | 'recording' | 'done' | 'failed'; export interface VideoAgentConfig { url: string; script: string; viewport: { width: number; height: number; }; /** Emulated device pixel ratio for this variant recording/capture. */ outputScale?: number; model: string; apiKey: string; /** Pipeline mode. 'clip' constrains the planner to micro-interactions (1-4 steps). */ mode?: 'video' | 'clip'; runId?: string; variantId?: string; projectId?: string; presetId?: string; theme?: 'light' | 'dark'; lang?: string; langInstructions?: string; themeInstructions?: string; navigationInstructions?: string; credentials?: LoginCredentials; maxPlanRetries?: number; maxStepRetries?: number; selectorMemory?: Record; targetMemory?: Record; preparedStartUrl?: string; preparedStorageState?: BrowserStorageState; preparedSessionStorage?: BrowserSessionStorageState; preparedObservationSummary?: string; preparedObservationSnapshot?: VideoObservationSnapshot; preparedActions?: ExecutedAction[]; preparedReplayActions?: ExecutedAction[]; preparedCoherenceKey?: string; videoOptions?: VideoRunOptions; /** Send a page screenshot to the planner LLM for visual grounding. Off by default. */ enablePlannerVision?: boolean; analyticsId?: string; /** Internal phase budget propagated by the clip orchestrator. */ internalPhaseTimeoutMs?: number; internalPhaseStartedAt?: number; abortSignal?: AbortSignal; /** Fallback model used when the primary model fails to produce tool calls (e.g. in repair lane). */ fallbackModel?: string; /** When true, the recording phase skips confirmPreparedVariantState because the dry-run already confirmed it. */ dryRunVariantConfirmed?: boolean; } export interface VideoAgentResult { success: boolean; plan: VideoPlan | null; videoPath: string | null; thumbnailBuffer: Buffer | null; durationMs: number; stepsExecuted: number; assessment: string; usage: StepUsage[]; } export interface ClipDefinition { /** Unique slug — identifies this clip in results and storage */ id: string; /** Display name shown in the UI */ name: string; /** * Natural-language description of the interaction to capture (the recording part). * When `navigationScript` is also provided, this field describes ONLY what should * be recorded — not how to get there. * When `navigationScript` is omitted, this field is used for both navigation and recording * (legacy behavior). */ script: string; /** * Optional per-clip navigation instructions: describes WHERE to navigate before recording. * Examples: "Navigate to the Acme Inc project Presets page", "Open the Settings tab". * * When provided, the navigation agent uses this to reach the correct page/state, * and `script` is used exclusively for planning the recorded interaction. * This separation eliminates ambiguity about what is navigation vs what is the recording. */ navigationScript?: string; /** Optional URL override — only needed when this clip starts at a different URL than the preset root */ url?: string; /** Seconds to freeze the last frame before the GIF loops. 0–10. Default: 0. */ holdLastFrameSec?: number; } export interface ClipOptions { /** Preferred delivery format. Studio still persists MP4 alongside GIF for editing/playback. */ format?: 'gif' | 'mp4' | 'both'; /** Max duration in seconds. Default: 8. Clips are trimmed if they exceed this. */ maxDurationSec?: number; /** GIF framerate. Default: 24. Lower = smaller file, less smooth. */ gifFps?: number; /** GIF max width in pixels. Default: 1440 (native viewport, no downsample for common 1440×900 presets). Downscaled from capture resolution above this to cap file size. */ gifMaxWidth?: number; /** Enable cursor animation in clips. Default: true. */ showCursor?: boolean; /** Cursor style. Default: 'minimal'. */ cursorTheme?: VideoCursorTheme; /** Whether to loop GIF output. Default: true. */ loop?: boolean; /** Add padding frames at start/end for smoother loop. Default: true. */ loopPadding?: boolean; /** Seconds to freeze the last frame before the GIF loops. 0–10. Default: 0. */ holdLastFrameSec?: number; /** Seconds to trim from the start of the recording (setup phase). Auto-calculated. */ trimStartSec?: number; /** Optional physical output width for the MP4 export. */ mp4Width?: number; /** Optional physical output height for the MP4 export. */ mp4Height?: number; /** Skip WebM→MP4 transcode (source is already MP4 from client CDP loop). */ skipMp4Encode?: boolean; } /** Usage metadata from a single OpenRouter API call */ export interface StepUsage { stepNumber: number; stepType: 'agent_iteration' | 'verification' | 'element_capture' | 'video_planning' | 'video_variant_classification' | 'video_step_verification' | 'video_step_fix' | 'assistant_chat' | 'studio_creation' | 'studio_iteration' | 'studio_capture_suggestion' | 'mock_data_generation' | 'page_identity_classification' | 'capture_verification' | 'alt_text_generation' | 'healer_invocation' | 'cron_feedback_classification' | 'tts_generation'; generationId: string | null; modelRequested: string; modelUsed: string | null; promptTokens: number | null; completionTokens: number | null; totalTokens: number | null; imagesInPrompt: number; /** Prompt assembly telemetry (optional, for cost optimization tracking) */ systemPromptChars?: number; toolSchemaChars?: number; userPayloadChars?: number; accessibilityChars?: number; interactiveElementCount?: number; actionHistoryCount?: number; elementsChars?: number; sessionSummaryChars?: number; selectorMemoryChars?: number; agentContextChars?: number; profileValidationStatus?: SessionProfileValidationStatus; sessionProfileSource?: SessionProfileSource; repairPathUsed?: string | null; evaluatorUsed?: boolean; cacheReadTokens?: number | null; cacheWriteTokens?: number | null; reasoningTokens?: number | null; sessionProfileReused?: boolean; actionReplayUsed?: boolean; } export type AgentLane = 'fast_nav' | 'full_nav' | 'repair' | 'verification'; export type ActionExpectedEffect = 'route_change' | 'dialog_open' | 'menu_open' | 'panel_expand' | 'selection_applied' | 'content_identity_change' | 'no_effect_ok'; export type VerificationReadinessState = 'ready' | 'not_ready_actionable' | 'unstable_retryable'; export type ReplayFailureCode = 'wrong_route' | 'wrong_identity' | 'missing_dialog' | 'unstable_loading' | 'anchor_unresolved' | 'late_replay_divergence' | 'duplicate_verification_skipped' | 'execution_failed' | 'verification_failed'; export interface AgentRunTelemetry { plannerCallCount: number; visionCallCount: number; verificationCallCount: number; imageUploadCount: number; verificationCacheHits: number; replayTrimmedActionCount: number; noEffectFailureCount: number; unstableRetryableCount: number; replayCheckpointMatchCount: number; replayCheckpointMissCount: number; replaySkipReason?: string | null; replayFailReason?: string | null; replayFailureCode?: ReplayFailureCode | null; phaseTimingsMs: Record; reasoningEffortUsed?: 'low' | 'medium' | 'high' | 'xhigh' | 'off'; usedLitePageState?: boolean; } export interface VerificationResult { verified: boolean; successVerdict?: CaptureSuccessVerdict; reason?: string; usage: StepUsage[]; fatal?: boolean; matchedPageId?: string | null; duplicateOfPageId?: string | null; blockingReason?: string; pageFingerprint?: string | null; checkpointEvidence?: CaptureCheckpointEvidence | null; mode?: 'deterministic' | 'vision' | 'text_fallback' | 'bailout'; readinessState?: VerificationReadinessState; } export interface SelectorValidationResult { matchCount: number; boundingBox: { x: number; y: number; width: number; height: number; } | null; visible: boolean; error?: 'no_match' | 'ambiguous' | 'invisible' | 'zero_size'; errorMessage?: string; } export interface ElementCaptureResult { element: IsolatedElement; success: boolean; buffer: Buffer; assessment: string; matchedIndex?: number; confidence?: number; capturedRegion?: { x: number; y: number; width: number; height: number; }; capturedSelector?: string; validation?: SelectorValidationResult; usage: StepUsage[]; } export interface LoginCredentials { loginUrl?: string; email?: string; password?: string; } export interface ObservedRequest { url: string; method: string; status: number; responseBody: unknown; } export interface ResolvedMock { urlPattern: string; method?: string; status: number; contentType: string; responseBody: unknown; } export interface AgentContextEntry { error_type: string; message: string; user_response: string | null; created_at: string; } export interface AgentConfig { url: string; prompt: string; promptFingerprint?: string; dark: boolean; langs: string[]; outputDir: string; headed: boolean; viewport: { width: number; height: number; }; maxIterations: number; model: string; fallbackModel?: string; /** * Optional dedicated vision model for screenshot analysis (dual-model architecture). * When set, screenshots are analyzed by this cheap vision model, and the main model * receives only text observations — eliminating image tokens from the main context. */ visionModel?: string; /** OpenRouter provider routing preferences per model ID. Looked up by model at call time. */ providerPreferences?: Record; credentials?: LoginCredentials; langInstructions?: string; themeInstructions?: string; currentLang?: string; currentTheme?: 'light' | 'dark'; viewports?: Array<{ width: number; height: number; }>; /** Language for the agent's reasoning/thinking output (e.g. "fr", "en") */ reasoningLocale?: string; /** OpenRouter reasoning effort for compatible models. "off" disables. Default: "medium". */ reasoningEffort?: 'low' | 'medium' | 'high' | 'xhigh' | 'off'; /** Set of model IDs whose OpenRouter `supported_parameters` includes "reasoning". Resolved at capture start. */ reasoningCapableModels?: Set; lane?: AgentLane; /** Past errors and user corrections, compressed into actionable hints. */ runHints?: AgentRunHint[]; /** Known selectors from previous successful runs, keyed by step signature */ selectorMemory?: Record; /** Validated persisted browser session/profile for this run, if available. */ sessionProfile?: ValidatedSessionProfile; /** Live browser state carried over from the previous capture in the same run. */ handoffContext?: CaptureHandoffContext; /** Multi-page manifest for the current variant/page. */ variantManifest?: VariantCaptureManifest; /** Internal run mode that tunes verification and replay behavior. */ runMode?: 'capture' | 'language_preflight' | 'repair' | 'video_navigation_preflight'; /** Explicit runtime objective for the current agent loop. */ currentObjective?: CaptureObjective; /** Current backend-owned cursor for the capture pipeline. */ captureCursor?: CaptureCursor; /** Active repair ticket when the agent is running as a focused repair sub-plan. */ activeRepairTicket?: RepairTicket | null; /** Remaining canonical capture queue, ordered and non-reorderable. */ remainingCaptureQueue?: string[]; /** Internal kill switches for rollout/debugging. */ enableDeterministicRecovery?: boolean; enableRecoveryEvaluator?: boolean; enableSalienceCompression?: boolean; /** Reference from variant 1's successful capture — used to guide subsequent variants * when cross-variant replay fails, so the LLM knows exactly what state to replicate. */ variantReference?: { finalUrl: string; assessment: string; pageTitle: string; /** Full action sequence from variant 1 — enables smarter cross-variant replay. */ actions?: ExecutedAction[]; }; /** Isolated elements to capture after the page is ready */ elements?: IsolatedElement[]; /** Analytics distinct ID to use for PostHog events (overrides the default machine-based ID) */ analyticsId?: string; abortSignal?: AbortSignal; /** * Called at each iteration boundary. If the user has requested a pause, * this callback awaits until guidance is submitted, then returns the message. * Returns null if no pause is pending. */ guidanceCallback?: () => Promise; verificationCache?: Map; runtimeTelemetry?: AgentRunTelemetry; uploadImageMode?: 'prefer_data_uri' | 'prefer_upload'; /** * Optional callback to upload a screenshot buffer to external storage and return a public URL. * When provided, image messages use HTTPS URLs instead of base64 data URIs — improving * compatibility with providers (e.g. xAI, Mistral) that reject data URIs. * Falls back to base64 if the callback throws. */ uploadImage?: (buffer: Buffer, mimeType: 'image/jpeg' | 'image/png') => Promise; } export interface InteractiveElement { index: number; tag: string; role: string; text: string; ariaLabel: string | null; /** HTML title attribute — important for icon buttons that have no visible text. */ title?: string | null; ariaControls?: string | null; ariaExpanded?: string | null; ariaHasPopup?: string | null; href: string | null; inputType: string | null; boundingBox: { x: number; y: number; width: number; height: number; } | null; selector: string; visible: boolean; visibilityState: 'full' | 'partial' | 'offscreen'; } export type AKType = 'button' | 'input' | 'link' | 'text' | 'image' | 'heading' | 'list' | 'table' | 'container' | 'icon' | 'toggle' | 'select' | 'checkbox' | 'radio' | 'slider' | 'tab' | 'modal' | 'nav' | 'form' | 'video' | 'audio' | 'iframe'; export type SemanticPattern = 'cookie-banner' | 'modal' | 'navbar' | 'footer' | 'hero' | 'sidebar' | 'dropdown' | 'toast' | 'tooltip' | 'form' | 'card' | 'pricing-table' | 'cta-group'; export type SemanticTrait = 'floating' | 'overlay' | 'sticky' | 'fixed' | 'above-fold' | 'below-fold' | 'full-width' | 'scrollable'; export interface AKBounds { x: number; y: number; w: number; h: number; } export interface ScrollState { scrollTop: number; scrollLeft: number; scrollHeight: number; scrollWidth: number; clientHeight: number; clientWidth: number; overflowY: boolean; overflowX: boolean; } export interface OverlayInfo { nodeId: string; zIndex: number; coveragePercent: number; blocksInteraction: boolean; } export interface OverlayScopeSummary { nodeId: string; kind: 'dialog' | 'menu' | 'popover' | 'dropdown' | 'overlay'; label: string; interactiveCount: number; subjectTokenOverlap: number; visibleLabels: string[]; summary: string; } export interface AKNode { id: string; type: AKType; label: string; value?: string; bounds: AKBounds; visible: boolean; interactive: boolean; state: { disabled: boolean; focused: boolean; checked?: boolean; expanded?: boolean; selected?: boolean; }; style: { bgColor?: string; fgColor?: string; fontSize?: number; opacity?: number; }; semantic: { pattern?: SemanticPattern; confidence: 'high' | 'medium'; traits: SemanticTrait[]; }; scroll?: ScrollState; attributes: Record; children: AKNode[]; sourceRef: string; } export interface AKPageState { url: string; title: string; viewport: { width: number; height: number; }; scroll: ScrollState; } export interface AKTree { root: AKNode; page: AKPageState; overlays: OverlayInfo[]; } export interface FocusQuery { type?: AKType[]; semantic?: SemanticPattern[]; trait?: SemanticTrait[]; interactive?: boolean; visible?: boolean; within?: string; labelContains?: string; maxDepth?: number; includeAncestors?: boolean; } export interface AKNodeRuntimeIndexEntry { id: string; sourceRef: string; bounds: AKBounds; type: AKType; label: string; interactive: boolean; visible: boolean; value?: string; attributes: Record; state: AKNode['state']; semantic: AKNode['semantic']; scroll?: ScrollState; } export interface PageState { cleanScreenshot: Buffer; screenshot: Buffer; akTree: AKTree; serializedAKTree: string; /** * Legacy observation fields retained temporarily for internal adapter paths. * New screenshot-agent logic must read `akTree` / `serializedAKTree` instead. */ accessibilityTree: string; interactiveElements: InteractiveElement[]; /** Simplified DOM: clean HTML structure without scripts/styles/classes, budget-capped. */ simplifiedDOM: string; scrollInfo: { scrollY: number; scrollHeight: number; viewportHeight: number; }; } /** Lightweight page state without screenshots — used when vision is not needed. */ export interface PageStateLite { akTree: AKTree; serializedAKTree: string; accessibilityTree: string; interactiveElements: InteractiveElement[]; simplifiedDOM: string; scrollInfo: { scrollY: number; scrollHeight: number; viewportHeight: number; }; } export type ActionType = 'tap' | 'type' | 'click' | 'type_text' | 'select_option' | 'scroll' | 'press_key' | 'wait' | 'dismiss_overlays' | 'search_text' | 'resize_viewport' | 'navigate_to' | 'analyze_screenshot' | 'capture' | 'focus' | 'take_screenshot' | 'ready_to_capture' | 'give_up' | 'note' | 'begin_subgoal' | 'capture_by_selector' | 'hover' | 'safe_expand' | 'scroll_to_element'; export interface ExecutedAction { iteration: number; action: ActionType; params: Record; success: boolean; error?: string; outcome?: string; stateChanged?: boolean; expectedEffect?: ActionExpectedEffect; observedEffects?: string[]; effectConfirmed?: boolean; noEffectReason?: string | null; failureClass?: string | null; origin?: CaptureActionOrigin; phase?: CaptureRunPhase; checkpointId?: string | null; } export interface WorkflowScreenshot { index: number; iteration: number; label: string; buffer: Buffer; path: string | null; } export interface WorkflowScreenshotEntry { index: number; iteration: number; label: string; filename: string; } export interface DiagnosticState { screenshot: Buffer; url: string; interactiveElements: InteractiveElement[]; accessibilityTreeSnippet: string; giveUpReason: string; } export interface AgentResult { success: boolean; captureCompleted?: boolean; captureNodeId?: string | null; screenshotPath: string | null; screenshots: WorkflowScreenshot[]; iterations: number; actions: ExecutedAction[]; assessment: string; diagnostic?: DiagnosticState; usage: StepUsage[]; runtimeStrategy?: 'full_llm' | 'action_replay' | 'action_replay_fallback' | 'preverified_handoff'; deterministicRecoveryUsed?: boolean; evaluatorUsed?: boolean; verification?: VerificationResult; telemetry?: AgentRunTelemetry; } export declare function isStrongCheckpointVerified(evidence?: CaptureCheckpointEvidence | null): boolean; export declare function isTerminalVerificationSuccess(verification?: VerificationResult | null): verification is VerificationResult & { verified: true; successVerdict?: 'terminal_match'; }; export declare function isTerminalAgentResultSuccess(agentResult?: Pick | null): agentResult is Pick & { success: true; captureCompleted?: true; }; export interface CaptureManifest { url: string; prompt: string; timestamp: string; captures: CaptureEntry[]; } export interface CaptureEntry { theme: 'light' | 'dark'; lang: string; filename: string; screenshots: WorkflowScreenshotEntry[]; result: AgentResult; }