import { type SafeFetchPolicy } from '../utils/safe-fetch.js';
import type { DocumentSource } from './index.js';
export type { SafeFetchPolicy } from '../utils/safe-fetch.js';
/**
 * Options for the web loaders. The fetch policy is the same one the `fetch_url` tool uses: private,
 * loopback, and cloud-metadata addresses are refused unless explicitly allowed, and every redirect
 * hop is checked again.
 */
export interface WebLoaderOptions extends SafeFetchPolicy {
    /** Largest response read, in bytes. Defaults to 5 MB. */
    maxBytes?: number;
    /** Redirects followed per page. Defaults to 5. */
    maxRedirects?: number;
    /** Time allowed per page, in milliseconds. Defaults to 15 seconds. */
    timeoutMs?: number;
    /** Pages fetched at once. Documents are still yielded in the order the URLs were given. Defaults to 4. */
    concurrency?: number;
    /** Stops loading. */
    signal?: AbortSignal;
    /** Application data added to every document. */
    metadata?: Record<string, unknown>;
    /** Keeps only a page's `<main>` or `<article>`, as `htmlToText()` does. On by default. */
    mainContent?: boolean;
    /**
     * What a page that cannot be loaded does: `throw` stops the load, `skip` passes over it and calls
     * `onSkip`. Defaults to `throw`.
     */
    onError?: 'throw' | 'skip';
    /** Called for each page skipped, with the reason. */
    onSkip?: (url: string, reason: Error) => void;
    /**
     * Renders a page whose content a script builds, with a browser of the application's own, such as
     * `browserRenderer()` over Playwright's: given the page fetched, it returns the HTML the browser ends
     * with. Called for an HTML page that runs a script and whose text, as fetched, is shorter than
     * `renderBelow`. The fetch has checked the page's URL first; the browser fetches what the page
     * loads under its own policy.
     */
    render?: WebPageRenderer;
    /** Renders a page whose fetched text is shorter than this many characters. Defaults to 200; `Infinity` renders every page that runs a script. */
    renderBelow?: number;
}
/** A page as fetched, which a renderer is given. */
export interface WebPageToRender {
    /** Its URL, after redirects. */
    url: string;
    /** The HTML the server sent. */
    html: string;
}
/**
 * Renders a page with a browser: the HTML it ends with once the page's scripts have run, which is
 * what a page built by script holds.
 */
export type WebPageRenderer = (page: WebPageToRender, signal: AbortSignal) => Promise<string> | string;
/** What `browserRenderer()` drives: a Playwright or Puppeteer browser, or anything shaped like one. */
export interface RenderingBrowser {
    /** Opens a page. */
    newPage(): Promise<{
        /** Goes to a URL and waits as told. */
        goto(url: string, options?: {
            waitUntil?: string;
            timeout?: number;
        }): Promise<unknown>;
        /** The HTML the page holds now. */
        content(): Promise<string>;
        /** Closes it. */
        close(): Promise<unknown>;
    }>;
}
/** Options for `browserRenderer()`. */
export interface BrowserRendererOptions {
    /** When the page counts as rendered. Defaults to `networkidle`, as Playwright names it. */
    waitUntil?: string;
    /** Time allowed per page, in milliseconds. Defaults to 30 seconds. */
    timeoutMs?: number;
}
/**
 * A renderer over a browser of the application's own, such as Playwright's `chromium.launch()`: each
 * page is opened, waited on, read, and closed. The package depends on no browser; the browser fetches
 * the page and what it loads under its own network policy, not the loader's.
 *
 * ```ts
 * const browser = await chromium.launch();
 * for await (const page of loadWebPages(urls, { render: browserRenderer(browser) })) store(page);
 * ```
 */
export declare function browserRenderer(browser: RenderingBrowser, options?: BrowserRendererOptions): WebPageRenderer;
/** Raised when a page cannot be loaded: a status that is not 2xx, or a type that is not text. */
export declare class WebLoaderError extends Error {
    /** The page. */
    readonly url: string;
    /** The HTTP status, when the server answered. */
    readonly status?: number | undefined;
    constructor(message: string, 
    /** The page. */
    url: string, 
    /** The HTTP status, when the server answered. */
    status?: number | undefined);
}
/**
 * Web pages, one document each, fetched through the SSRF-safe fetch. HTML becomes text through
 * `htmlToText()`, with its title and description in the metadata; plain text, Markdown, and JSON are
 * kept as they are. The id is the final URL after redirects.
 */
export declare function loadWebPages(urls: Iterable<string | URL> | AsyncIterable<string | URL>, options?: WebLoaderOptions): AsyncGenerator<DocumentSource>;
/** Options for `sitemapUrls()` and `loadSitemap()`. */
export interface SitemapOptions extends WebLoaderOptions {
    /** Keeps a page URL; the rest are not fetched. */
    include?: (url: string) => boolean;
    /** Most page URLs returned. Defaults to 1,000. */
    limit?: number;
}
/**
 * The page URLs a sitemap lists. A sitemap index is followed into its sitemaps, three levels deep at
 * most, and gzipped sitemaps are decompressed. Each URL appears once.
 */
export declare function sitemapUrls(url: string | URL, options?: SitemapOptions): Promise<string[]>;
/** Every page a sitemap lists, loaded as `loadWebPages()` loads them. */
export declare function loadSitemap(url: string | URL, options?: SitemapOptions): AsyncGenerator<DocumentSource>;