export declare const HEADER_USER_AGENT = "User-Agent";
export declare const ROBOTS_TXT_FILENAME = "robots.txt";
export declare const PROTOCOL_HTTP = "http:";
export declare const PROTOCOL_HTTPS = "https:";
export declare const ALLOW_ALL_ROBOTS_TXT_CONTENT = "User-agent: *\nAllow: /";