/**
 * ### Tardigrade the Web Crawler
 * <img src="https://i.imgur.com/iuzpcvD.png" width="350px" />
 *
 * 1. **Use Fetch API, check for bot detection.** Scrape  any domain's URL to get its HTML, JSON, or arraybuffer.<br />
 * Scraping internet pages is a [free speech right
 * ](https://blog.apify.com/is-web-scraping-legal/).
 * 2. Features: timeout, redirects, default UA, referer as google, and bot
 * detection checking. <br />
 * 3. If fetch method does not get needed HTML, use Docker proxy as backup.
 *
 * 4. [Setup Docker](https://github.com/vtempest/ai-research-agent/tree/master/src/crawler)
 *  container with NodeJS server API renders with puppeteer DOM to get all HTML loaded by
 *  secondary in-page API requests after the initial page request, including user login and cookie storage.
 * 5. Bypass Cloudflare bot check: A webpage proxy that request through Chromium (puppeteer) - can be used
 * to bypass Cloudflare anti bot using cookie id javascript method.
 * 6. Send your request to the server with the port 3000 and add your URL to the "url"
 *  query string like this: `http://localhost:3000/?url=https://example.org`
 *
 * 7. Optional: Setup residential IP proxy to access sites that IP-block datacenters
 *  and manage rotation with [Scrapoxy](https://scrapoxy.io). Recommended:
 * [Hypeproxy](https://hypeproxy.io/products/static-residential-proxies)
 * [NinjasProxy](https://ninjasproxy.com/residential-proxies/)
 * [Proxy-Cheap](https://app.proxy-cheap.com/order)
 * [LiveProxies](https://liveproxies.io/rotating-residential-proxies-pricing)
 *
 * @param {string} url - any domain's URL
 * @param {Object} [options]
 * @param {number} options.timeout default=5 -  abort request if not retrived, in seconds
 * @param {number} options.maxRedirects default=3 - max redirects to follow
 * @param {number} options.checkBotDetection default=true - check for bot detection messages
 * @param {number} options.changeReferer default=true - set referer as google
 * @param {number} options.userAgentIndex default=0 - index of [google bot, default chrome]
 * @param {string} options.proxy default=false - use proxy url
 * @param {boolean} options.checkRobotsAllowed default=false - check robots.txt rules
 * @returns {Promise<string>} -  HTML, JSON, arraybuffer, or error object
 * @category Extract
 * @example await scrapeURL("https://hckrnews.com", {timeout: 5, userAgentIndex: 1})
 * @author [vtempest (2025)](https://github.com/vtempest)
 */
export declare function scrapeURL(url: any, options?: {}): Promise<any>;
/**
 * As backup, scrape with JINA to get html
 * @param {string} url
 * @returns {Promise<string>}
 */
export declare function scrapeJINA(url: any): Promise<any>;
/**
 * Fetches and parses the robots.txt file for a given URL.
 * @param {string} url - The base URL to fetch the robots.txt from.
 * @returns {Promise<Object>} A JSON object representing the parsed robots.txt.
 */
export declare function fetchScrapingRules(url: any): Promise<{
    directives: {};
    crawlDelay: {};
    sitemaps: any[];
    preferredHost: any;
} | {
    error: string;
}>;