import robotsParser from 'robots-parser';
/** Fetch and parse robots.txt for a given base URL */
export declare function getRobotsParser(baseUrl: string): Promise<import("robots-parser").Robot | null>;
/** Check if a URL is allowed by robots.txt */
export declare function isAllowed(robots: ReturnType<typeof robotsParser> | null, url: string): boolean;