import { type BrowserContext, type Page } from 'playwright'; import type { AKNode, AKNodeRuntimeIndexEntry, AKTree, BrowserOptions, BrowserSessionStorageState, BrowserStorageState, InteractiveElement, ObservedRequest, OutscaleConfig, PageState, PageStateLite, ResolvedMock, SelectorValidationResult, VideoPageSignals } from './types.js'; import type { ProgressSnapshot, VisualStabilityResult } from './execution-types.js'; export interface SelectorCaptureError { error: 'no_match' | 'ambiguous' | 'invisible' | 'zero_size'; errorMessage: string; } export interface StableSelectorSnapshot { tag: string; id?: string | null; role?: string | null; type?: string | null; name?: string | null; href?: string | null; ariaLabel?: string | null; title?: string | null; placeholder?: string | null; dataAk?: string | null; dataAkInteract?: string | null; dataTestId?: string | null; dataTest?: string | null; dataQa?: string | null; dataCy?: string | null; dataLang?: string | null; dataLanguage?: string | null; dataLocale?: string | null; dataTheme?: string | null; dataColorScheme?: string | null; hreflang?: string | null; } export declare function buildStableCssSelectorCandidates(snapshot: StableSelectorSnapshot): string[]; export declare function selectStableCssSelector(snapshot: StableSelectorSnapshot, isUniqueSelector: (selector: string) => boolean): string | null; type VisibilityState = InteractiveElement['visibilityState']; interface ElementBox { x: number; y: number; width: number; height: number; } export interface BrowserObservation { url: string; title: string; readyState: string; scrollX: number; scrollY: number; scrollHeight: number; interactiveCount: number; dialogCount: number; expandedCount: number; loadingIndicatorCount: number; textSample: string; surfaceSignature: string; primarySurface: string; overlaySurface: string | null; navigationSurface: string; configurationSurface: string | null; hasBlockingOverlay: boolean; strongSurfaceSignature: string; } export interface BrowserReaction { before: BrowserObservation; after: BrowserObservation; changed: boolean; summary: string; } export interface BrowserVerificationBundle { screenshot: Buffer; pageSignals: VideoPageSignals; observation: BrowserObservation; url: string; title: string; capturedAt: string; coherenceKey: string; } export interface BrowserVideoVerificationBundle extends BrowserVerificationBundle { accessibilityTree: string; interactiveElements: InteractiveElement[]; scrollInfo: PageState['scrollInfo']; } export interface BrowserSelectorProbe { tag: string; role: string; href: string | null; label: string; inputType: string | null; ariaExpanded?: string | null; ariaControls?: string | null; ariaHasPopup?: string | null; } export interface BrowserStorageHintWriteParams { storageName: 'localStorage' | 'sessionStorage'; key: string; candidate: string; kind: 'locale' | 'theme'; } export declare function describeObservationChange(before: BrowserObservation, after: BrowserObservation): BrowserReaction; export declare class Browser { private options; private browser; private context; private page; private elementMap; private akNodeIndex; private poolContext; private persistentContext; private ownedChromiumProfileDir; /** * Network-progress counters (AUT-240, Layer C). Maintained by request * lifecycle listeners attached lazily and idempotently per page via * `ensureProgressListeners()`, so every page-creation path is covered without * touching each one. Count FIRST-PARTY traffic only (same site as the live * main-frame origin) so background third-party telemetry does not read as * progress. Read cheaply by `getProgressSnapshot()`. */ private progressListenersPage; private inflightRequests; private networkEventCount; private lastNetworkActivityAtMs; /** * Xvfb instance backing the headed Chromium used by clip capture on Cloud * Run with NVIDIA L4. Set when forClipCapture spawns Xvfb; null otherwise * (Mac/Win and local Linux use headless Playwright). Lifetime matches the * browser process — closeContext() leaves it alone, close() tears it down. */ private xvfb; /** Public read access for the clip recorder (needs the DISPLAY string). */ get xvfbDisplay(): string | null; /** Viewport size used for clip capture region (matches Xvfb screen). */ get viewport(): { width: number; height: number; }; constructor(options: BrowserOptions); /** * Create a Browser using the shared pool (server/web API mode). * The Chromium process is reused across captures; only the context is isolated. * Call browser.close() when done — it releases the context back to the pool. */ static fromPool(options: BrowserOptions): Promise; /** * Create a Browser dedicated to clip capture. Frames are pulled via CDP * `Page.captureScreenshot` in a tight loop by `ClipCaptureLoop` — NOT via * Playwright's built-in `recordVideo` (which plateaus at 27 FPS with ~12% * duplicates due to the CDP screencast throttler, and on cloud Linux runs * the WebM encoder on the compositor thread → frame drops to ~4 fps under * software rasterization). * * Preserves the HiDPI rendering path (`--force-device-scale-factor` + * `--window-size`) so the captured frames match viewport × DSF pixels, and * the cursor overlay script so clicks/hover moments are visible. */ static forClipCapture(options: BrowserOptions, cursorScript: string): Promise; /** * Close the active capture context promptly after recording stops. Regular * clip capture leaves the browser process alive for teardown; persistent * Cloud Run contexts close their owned browser process with the context. * Call browser.close() afterwards to stop Xvfb and remove owned temp state. */ closeContext(): Promise; launch(): Promise; /** Wires debug-only lifecycle listeners on the page/context/browser to surface * navigations, console errors, page crashes, and unexpected closures. * No-op when debug logging is disabled. */ private attachDebugLifecycleListeners; recreateContext(options?: Partial>): Promise; setDeviceScaleFactor(deviceScaleFactor: number): Promise; addCookies(cookies: Array<{ name: string; value: string; domain: string; path?: string; httpOnly?: boolean; secure?: boolean; sameSite?: 'Lax' | 'None' | 'Strict'; }>): Promise; close(): Promise; navigateTo(url: string): Promise; /** * Wait for DOM to stabilize: no MutationObserver events for `settleMs`, * with an overall timeout of `timeoutMs`. */ private waitForDomStability; private waitForFontsBeforeScreenshot; /** * Attach network-progress listeners to `page` once. Idempotent per page: a * no-op if already attached, and resets counters only when the `Page` object * itself changes (recreated context, pooled re-acquire). A `page.reload()` * keeps the same object, so counters stay monotonic across it — harmless, as * the watchdog only ever compares deltas via `hasProgress`. Covers every * page-creation path without editing each one. */ private ensureProgressListeners; /** * Cheap snapshot of page activity for the runner's progress watchdog * (AUT-240, Layer C). Network counters are free (maintained by listeners); * `readyState`/`domNodeCount` come from one light `evaluate`. Never rejects: * an `evaluate` failure (navigation in flight) is itself progress, surfaced * as `navigating: true`. */ getProgressSnapshot(): Promise; /** * Wait until the page is visually stable enough to screenshot (AUT-240, * Layer B). Best-effort and non-blocking: composes light semantic signals — * fonts ready, images loaded, no visible `[aria-busy]`/`[role=progressbar]`, * DOM quiet — and only falls back to a bounded pixel-convergence check when * loaders never clear. Returns `stable: false` with a reason instead of * throwing or blocking; the caller captures anyway (a perpetual animation is * cosmetic, not a reason to fail the capture — decision 1). */ waitForVisuallyStable(options?: { maxWaitMs?: number; }): Promise; private remainingMs; /** Poll until in-DOM images have finished loading, or the budget elapses. */ private waitForImagesSettled; /** * Poll until no semantically-marked loader is visible, or the deadline hits. * Condition: zero visible elements matching * `[aria-busy="true"], [role="progressbar"]:not([hidden])` — covers shadcn, * Radix, MUI, etc. without a hardcoded class list. Returns true if cleared. */ private waitForNoVisibleLoaders; /** * Bounded pixel-convergence fallback (AUT-240 decision 1): take up to * `PIXEL_FALLBACK_MAX_PASSES` raw frames; if two consecutive frames are within * `PIXEL_FALLBACK_DIFF_THRESHOLD`, the page is settled. Never blocks past the * deadline; returns false (capture anyway) for perpetual animations. */ private pixelConvergenceFallback; takeScreenshot(): Promise; takeScreenshotForAI(options?: { timeoutMs?: number; }): Promise; getAccessibilityTree(options?: { timeoutMs?: number; }): Promise; getInteractiveElements(options?: { timeoutMs?: number; }): Promise; /** * Extract a simplified DOM representation of the page. * Strips scripts, styles, SVGs, class/style attributes, and collapses empty containers. * Returns clean indented HTML-like text, budget-capped at ~4000 chars. */ getSimplifiedDOM(): Promise; private mapRawNodeType; private convertRawNode; private shouldCollapseNode; private collapseNode; private collectFingerprintEntries; private assignFingerprints; private applyTraitAnnotations; private findFirstPattern; private flattenNodes; private annotateSemanticPatterns; private computeOverlays; private normalizeRawSnapshot; getAKTree(): Promise; getPageState(opts?: { skipAnnotation?: boolean; }): Promise; /** Lightweight page state without screenshots — skips takeScreenshotForAI + SoM annotation. */ getPageStateLite(): Promise; exportStorageState(): Promise; exportSessionStorage(): Promise; reloadCurrentPage(options?: { waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit'; timeout?: number; }): Promise; writeStorageHintCandidate(params: BrowserStorageHintWriteParams): Promise; probeSelector(selector: string): Promise; prepareSessionStorage(bundle: BrowserSessionStorageState | undefined, options?: { replace?: boolean; }): Promise; capturePageSignals(options?: { timeoutMs?: number; }): Promise; captureObservation(): Promise; captureVerificationBundle(options?: { settleMs?: number; maxAttempts?: number; }): Promise; captureVideoVerificationBundle(options?: { settleMs?: number; maxAttempts?: number; onRetry?: (message: string) => void; helperTimeoutMs?: number; }): Promise; waitForPageReaction(before?: BrowserObservation, options?: { timeoutMs?: number; settleMs?: number; idleGraceMs?: number; }): Promise; private computeVisibilityState; private getVisiblePoint; private runClickHitTest; private withResolvedLocator; private measureElement; private ensureElementInView; private alignElementInView; scrollElementIntoView(index: number, options?: { align?: 'start' | 'center' | 'end'; margin?: number; }): Promise; dismissOverlays(): Promise<{ dismissed: boolean; method: string | null; }>; hoverByIndex(index: number): Promise; hoverBySelector(selector: string): Promise; hoverByCoordinates(x: number, y: number): Promise; safeExpand(options: { index?: number; selector?: string; expectedText?: string; expectedSelector?: string; }): Promise; clickByIndex(index: number): Promise; clickBySelector(selector: string, options?: { force?: boolean; }): Promise; private resolveActivationTarget; private activateSelectorTarget; clickByCoordinates(x: number, y: number): Promise; private buildElementSnapshotFromAKEntry; private activateAKNodeEntry; private clickAKNodeEntryByBounds; typeText(text: string, options?: { index?: number; selector?: string; clearFirst?: boolean; }): Promise; selectOption(options: { index?: number; selector?: string; optionLabel?: string; optionValue?: string; optionIndex?: number; }): Promise; scroll(direction: string, amount?: number, selector?: string): Promise; centerNodeInView(centerNodeId: string, options?: { containerNodeId?: string; offset?: number; }): Promise; pressKey(key: string): Promise; searchText(query: string): Promise>; /** Temporarily hide all fixed/sticky overlays (navbars, banners, FABs, etc.) */ private hideFixedOverlays; /** Restore overlays hidden by hideFixedOverlays */ private restoreFixedOverlays; screenshotElement(index: number, padding?: number): Promise; screenshotRegion(x: number, y: number, width: number, height: number, padding?: number): Promise; screenshotBySelector(selector: string, outscale?: OutscaleConfig | number): Promise<{ buffer: Buffer; validation: SelectorValidationResult; }>; /** * Capture a page region by bounding box coordinates (document-relative). * Used as a fallback when no CSS selector can be resolved for the target element. */ screenshotByRegion(region: { x: number; y: number; width: number; height: number; }, outscale?: OutscaleConfig | number): Promise; resolveAKNode(nodeId: string, options?: { reparse?: boolean; }): Promise; tapNode(nodeId: string, options?: { force?: boolean; }): Promise; typeIntoNode(nodeId: string, text: string, options?: { clearFirst?: boolean; }): Promise; captureNode(nodeId?: string): Promise; focusTree(query: import('./types.js').FocusQuery): Promise<{ tree: AKTree; serialized: string; matches: AKNode[]; }>; wait(ms: number): Promise; /** * Force-load all lazy-loaded images on the page before element capture. * * 1. Strip loading="lazy" and promote data-src/data-srcset attributes * 2. Scroll through the full page to trigger IntersectionObserver callbacks * 3. Wait for all visible images to finish loading */ forceLoadLazyImages(options?: { timeout?: number; }): Promise; setColorScheme(scheme: 'light' | 'dark'): Promise; setLanguage(lang: string): Promise; resizeViewport(width: number, height: number): Promise; get currentPage(): Page; get browserContext(): BrowserContext; /** * Observation pass for mock data generation. * Navigates to the given URL, waits for network idle, and records all JSON API responses. * Auth/analytics/asset endpoints are filtered out automatically. */ observeNetworkRequests(url: string, waitMs?: number): Promise; /** * Sets up Playwright route interceptors for each resolved mock. * Must be called before navigating to the page for the actual capture. */ setupRouteInterception(mocks: ResolvedMock[]): Promise; /** * Removes all route interceptors set by setupRouteInterception. * Must be called before setting up new mocks to prevent stacking. */ clearRouteInterception(): Promise; private ensurePage; private ensureContext; private buildContextOptions; } export {};