import { IRobotsDataService, CachedRobot } from '../../types';
export declare class RobotsDataService implements IRobotsDataService {
    private cache;
    /**
     * Retrieves the cached robot rules for the given URL's origin.
     * Fetches from the network if not already cached.
     */
    getRobot(url: string, userAgent?: string): Promise<CachedRobot>;
    /**
     * Updates the last crawled timestamp for the given URL's origin.
     */
    setLastCrawled(url: string, timestamp: number): void;
    private fetchRobotsTxt;
    private isUnavailable;
}