export interface RobotsCheckResult {
    allowed: boolean;
    reason?: "disallowed" | "fetch_failed" | "no_robots_txt";
}
interface CachedRobots {
    body: string | null;
    fetched: string;
}
export declare function getRobotsForOrigin(origin: string): Promise<CachedRobots>;
export declare function checkRobots(url: string, userAgent: string): Promise<RobotsCheckResult>;
export {};