import type { FetchOutput, CrawlInput, CrawlOutput, RawFetchResult } from '../types.js';
import { RobotsParser } from './robots.js';
export type FetchFn = (url: string) => Promise<FetchOutput>;
export type RawFetchFn = (url: string) => Promise<RawFetchResult>;
export declare class Crawler {
    private fetchFn;
    private rawFetchFn;
    private rateLimiter;
    constructor(fetchFn: FetchFn, rawFetchFn: RawFetchFn);
    crawl(input: CrawlInput): Promise<CrawlOutput>;
    private robotsTxtContent;
    private fetchRobots;
    private crawlTraversal;
    private filterLinks;
    crawlSitemap(input: CrawlInput, seedOrigin: string, maxPages: number, robotsParser: RobotsParser | null): Promise<CrawlOutput>;
    /**
     * Crawl an explicit list of URLs (e.g. from a sitemap probe). Applies
     * include/exclude patterns, robots.txt, max_pages, and rate limits the
     * same way as crawlSitemap.
     */
    private crawlFromExplicitUrls;
    private discoverSitemapUrls;
}
//# sourceMappingURL=crawler.d.ts.map