/** * ZeTa-AI Crawler — Stagehand hybrid architecture * * Three-layer stack: * Layer 1 — Playwright (deterministic): navigation, screenshots, DOM fallbacks * Layer 2 — Stagehand act/extract (AI): login, element understanding, smart link discovery * Layer 3 — Skyvern (vision): fallback for CAPTCHAs, legacy apps, visual-only flows * * enableCaching: true → repeat crawls of the same site cost ~zero AI tokens after first run. */ import type { Credentials } from './auth.js'; import { type SkyvernConfig } from './skyvern.js'; import { type TraversalStrategy } from './frontier.js'; import { type ApiEndpointObservationInput } from './network-observer.js'; import type { ProxyTierConfig } from './proxy.js'; interface PerformanceMetric { url: string; screenName: string; lcp?: number; fcp?: number; ttfb?: number; cls?: number; } /** * Lightweight pre-crawl link discovery: loads the app URL in a headless (proxy-aware) browser * and returns same-site `` links. Complements sitemap discovery for SPA/app sites that * have no sitemap.xml. Best-effort — returns [] on any failure (bot wall, timeout, etc.). * chromium.launch({ proxy }) handles proxy 407 auth itself (same client), so no CDP handler needed. */ export declare function discoverLinksViaBrowser(appUrl: string, opts?: { proxyUrl?: string; maxLinks?: number; }): Promise; export declare function crawlProject(opts: { tenantId: string; projectId: string; appUrl: string; /** The viewport used for this crawl pass */ viewport?: 'DESKTOP' | 'MOBILE_PORTRAIT' | 'TABLET'; credentials?: Credentials; /** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */ storageState?: string; screenshotDir?: string; /** Cloud storage config for immediate screenshot upload. If set, screenshots are uploaded to cloud immediately after capture and cloudUrl is persisted. */ storageConfig?: { provider: string; bucket: string; region?: string; accountId?: string; endpointUrl?: string; accessKeyId?: string; secretAccessKey?: string; prefix?: string; publicUrl?: string; } | null; /** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */ discoveryMode?: boolean; maxScreens?: number; /** Seed URLs added to the initial crawl queue — link discovery still runs from each page. */ startUrls?: string[]; /** When provided, crawl ONLY these exact URLs (no link-following, no full screen clear). Used for selected-screen recrawl. */ selectiveUrls?: string[]; /** When true, run the interactive probe after static crawl to capture real validation errors, toasts, and modals. */ interactive?: boolean; /** Optional abort signal — abort() cancels the crawl between page navigations. */ abortSignal?: AbortSignal; /** Platform job ID — stored on screens so we can diff which were found in each crawl. */ jobId?: string; /** Device profiles to capture screenshots for after the main DESKTOP crawl. * Values: 'MOBILE_PORTRAIT' | 'TABLET' * The main crawl always runs as DESKTOP. Extra profiles trigger a screenshot-only pass. */ deviceProfiles?: string[]; /** HTTP Basic Auth credentials — injected as Authorization header on same-origin requests. */ httpBasicUsername?: string; httpBasicPassword?: string; /** Bearer/API token — injected as a request header and into localStorage. Skips AI login. */ authToken?: string; /** Header name for authToken (default: 'Authorization'). */ authTokenHeader?: string; /** Prefix for authToken value (default: 'Bearer '). */ authTokenPrefix?: string; /** CAPTCHA solving API key (2captcha or CapSolver). */ captchaSolverApiKey?: string; /** CAPTCHA solving service provider (default: '2captcha'). */ captchaSolverProvider?: '2captcha' | 'capsolver'; /** * Path to a Chrome user data directory for profile inheritance. * When set, the crawler exports the existing session from this profile and injects it, * so the agent is already logged in. Works for local/self-hosted deployments only. * Ignored on cloud/Docker (headless env cannot open another Chrome instance safely). */ chromeProfilePath?: string; /** MailSlurp API key for email OTP auto-retrieval (GAP 5). */ mailslurpApiKey?: string; /** MailSlurp inbox ID to poll for OTP emails (GAP 5). */ mailslurpInboxId?: string; /** SSO provider for ROPC token flow — 'okta' | 'azure_ad' | 'generic_oidc' (GAP 18). */ ssoProvider?: string; /** SSO domain or Azure tenant ID (GAP 18). */ ssoDomain?: string; /** SSO OAuth2 client ID (GAP 18). */ ssoClientId?: string; /** SSO OAuth2 client secret (GAP 18). */ ssoClientSecret?: string; /** SSO scopes (space-separated, default: 'openid profile') (GAP 18). */ ssoScope?: string; /** HTTP/HTTPS/SOCKS5 proxy URL — e.g. 'http://user:pass@proxy.example.com:8080' or 'socks5://proxy:1080'. * Useful for sites that block datacenter IPs (Akamai, Cloudflare Bot Manager). */ proxyUrl?: string; /** Tiered proxy configuration for automatic per-domain escalation (free → datacenter → residential → premium). */ proxyTiers?: ProxyTierConfig[]; /** Use Camoufox (Firefox) as a cookie bootstrapper before the main Chromium crawl. * Camoufox visits the URL first to solve Cloudflare JS challenges and extract session * cookies, which are then injected into the Chromium crawl as storageState. * Enable via Project Settings → Crawl Config → useFirefox: true. */ useFirefox?: boolean; /** Natural language login instructions for Stagehand AI login. E.g. "Click Sign In in the nav, fill email, click Continue". */ loginInstructions?: string; /** Skyvern vision fallback config from workspace settings (overrides SKYVERN_API_URL/KEY env vars). */ skyvernConfig?: SkyvernConfig; /** * Max screens to crawl per segment (default 50). * After this limit the crawl pauses and remaining URLs are saved so the * next job can continue. Total crawl is unlimited — just runs in multiple jobs. */ segmentSize?: number; /** URLs still queued from a previous segment — injected by the job-queue auto-continuation. */ resumeUrls?: string[]; /** Segment number (1-based) — used for progress display only. */ segmentIndex?: number; /** Seed the crawl queue from the project's sitemap.xml, in addition to link-following. */ useSitemap?: boolean; /** Explicit sitemap URL override — defaults to `${origin}/sitemap.xml` when useSitemap is set. */ sitemapUrl?: string; /** Crawl queue traversal order. Default BFS — unchanged behavior from before this option existed. */ traversalStrategy?: TraversalStrategy; /** BEST_FIRST only: URLs containing any of these (case-insensitive) are prioritized. */ bestFirstKeywords?: string[]; /** When true, skip URLs disallowed by the site's robots.txt. Default false. */ respectRobotsTxt?: boolean; /** When true, dynamically slow crawl speed when server response latency is high (Scrapy-style AutoThrottle). Default false. */ autoThrottle?: boolean; /** Regex pattern — only crawl URLs whose path+query match this pattern. */ urlFilter?: string; /** Already-captured URLs from a previous failed job — pre-seeded into visited map so they're never re-crawled. */ skipUrls?: string[]; /** DOM hashes collected in a prior segment — prevents re-screenshotting structurally identical pages across segments. */ knownDomHashes?: string[]; /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */ onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise; /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */ onUrlsDiscovered?: (urls: string[]) => Promise; }, onProgress?: (pct: number, step: string) => Promise): Promise<{ discoveredUrls: string[]; screensFound: number; screensErrored: number; screensSkippedToNextSegment: number; isPartial: boolean; discovered: never[]; apiEndpoints: never[]; performanceMetrics: never[]; pendingUrls: never[]; segmentIndex: number; hasMoreSegments: boolean; domHashes: string[]; visitedUrls?: undefined; wasTimeout?: undefined; crawlWarnings?: undefined; } | { screensFound: number; screensErrored: number; screensSkippedToNextSegment: number; isPartial: boolean; discovered: { url: string; screenId?: string; elements?: number; elementFacts?: Array<{ meaning: string; role: string; expectedData?: string; }>; screenshotPath?: string; version?: number; error?: string; }[]; apiEndpoints: ApiEndpointObservationInput[]; performanceMetrics: PerformanceMetric[]; pendingUrls: string[]; visitedUrls: string[]; wasTimeout: boolean; segmentIndex: number; hasMoreSegments: boolean; domHashes: string[]; crawlWarnings: string[] | undefined; discoveredUrls?: undefined; }>; export declare function domFallbackElements(page: any, rawPage?: any): Promise; ariaState?: Record; parentLandmark?: string; }>>; export {};