import { ProxyAgent } from 'undici';
/**
 * Same-site check that tolerates www/non-www and http/https differences — a sitemap on
 * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).
 */
export declare function sameSite(candidateUrl: string, siteOrigin: string): boolean;
/**
 * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap
 * document (either a <urlset> or a <sitemapindex>), filtered to same-origin
 * and deduped. Pure function — no I/O — for direct unit testing without a
 * real HTTP fetch.
 */
export declare function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls?: number): {
    pageUrls: string[];
    subSitemapUrls: string[];
};
/**
 * Fetches and parses a sitemap (or sitemap index, recursed up to a small
 * depth) into a flat, deduped list of same-origin URLs.
 */
export declare function fetchSitemapUrls(sitemapUrl: string, origin: string, depth?: number, dispatcher?: any, maxUrls?: number): Promise<string[]>;
/**
 * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
 * explicit override) and returns the URLs it lists. Never throws.
 */
export declare function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]>;
/**
 * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because
 * DataImpulse-style usernames contain commas (country codes) that break new URL().
 * Returns undefined for no/auto proxy — the caller then fetches directly.
 */
export declare function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined;
/** Extract same-origin `Sitemap:` directives from a robots.txt body. */
export declare function parseRobotsSitemaps(robotsTxt: string, origin: string): string[];
export interface UrlCandidate {
    url: string;
    source: 'sitemap' | 'robots' | 'link';
}
/**
 * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap
 * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped
 * candidate list the user can review and seed before launching the Playwright crawl.
 * Never throws — bot-protected sites (Akamai/Cloudflare) simply yield an empty list.
 */
export declare function discoverProjectUrls(appUrl: string, opts?: {
    sitemapUrl?: string;
    proxyUrl?: string;
}): Promise<{
    candidates: UrlCandidate[];
    sitemapFound: boolean;
}>;