import { Robot } from 'robots-parser';
/**
 * Maintains a collection of Robot Exclusion rules for each site encountered
 * during a crawl.
 */
export declare class Robots {
    protected static _rules: Map<string, Robot>;
    protected static getHost(url: string | URL): string;
    static get rules(): Map<string, Robot>;
    static setRules(url: string | URL, rules: string): void;
    static getRules(url: string | URL): Robot | undefined;
    static isAllowed(url: string | URL, ua?: string): boolean | undefined;
    static isDisallowed(url: string, ua?: string): boolean | undefined;
    static getCrawlDelay(url: string | URL, ua?: string): number | undefined;
    static getSitemaps(url?: string | URL): string[];
    static getPreferredHost(url: string | URL): string | undefined;
    private constructor();
}
//# sourceMappingURL=robots.d.ts.map