/**
 * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
 *
 * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
 * No browser overhead. Falls back to browser engine on JS-heavy sites.
 *
 * Features:
 *   - HTTP/2 native (Node 20 built-in fetch)
 *   - ETag / Last-Modified response caching
 *   - 429 / 503 exponential backoff with jitter
 *   - BFS / DFS traversal
 *   - URL regex filtering
 *   - Robots.txt compliance
 *   - Link discovery (href, action, src)
 *   - Markdown extraction (Readability + Turndown)
 *   - OG / schema.org / meta extraction
 *   - Table → JSON extraction
 *   - JSON Lines, CSV, raw HTML output formats
 *   - Streaming progress via onProgress callback
 */
export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
export interface HttpScreen {
    url: string;
    title: string | null;
    markdown: string;
    html?: string;
    metadata: {
        description: string | null;
        ogTitle: string | null;
        ogDescription: string | null;
        ogImage: string | null;
        canonical: string | null;
        schemaTypes: string[];
        lang: string | null;
    };
    tables: Array<Record<string, string>[]>;
    links: string[];
    images: Array<{
        src: string;
        alt: string;
    }>;
    statusCode: number;
    fromCache: boolean;
    requiresBrowser?: boolean;
    durationMs: number;
}
export interface HttpCrawlOpts {
    appUrl: string;
    maxScreens?: number;
    urlFilter?: string;
    traversalStrategy?: 'BFS' | 'DFS';
    outputFormats?: HttpOutputFormat[];
    cacheDir?: string;
    userAgent?: string;
    headers?: Record<string, string>;
    respectRobotsTxt?: boolean;
    includeRawHtml?: boolean;
    timeoutMs?: number;
    concurrency?: number;
    onProgress?: (update: {
        url: string;
        screensFound: number;
        total: number;
        cached: boolean;
    }) => void;
}
export interface HttpCrawlResult {
    screens: HttpScreen[];
    stats: {
        total: number;
        cached: number;
        errors: number;
        rateLimited: number;
        durationMs: number;
        cacheHitRate: number;
    };
    jsonLines?: string;
    csv?: string;
}
export declare function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult>;