/** @module url-frontier
 *
 * Resources:
 * https://www.youtube.com/watch?v=BKZxZwUgL3Y
 * https://nlp.stanford.edu/IR-book/html/htmledition/the-url-frontier-1.html
 * https://nlp.stanford.edu/IR-book/
 */
import { AsyncMap } from "../utils/AsyncMap";
import { AsyncQueue } from "../utils/AsyncQueue";
import { HostHeap } from "./HostHeap";
/** A URL */
export declare type Link = string;
export declare function index(text: string): Promise<string>;
/** The host Cache */
/** Links come to the front queue first...they are ordered by priority and are enqueued from the prioritizer into this
 * frontQueue. The frontQueue is a list of queues numbered 1 -> F who help with prioritization of the links.
 * Links are added to smaller numbers (1 being the smallest) to signify they should take priority
 * over other links in higher numbered queues (with F being the highest).
 *
 * Note that there should only ever be one frontQueue (meaning one list of queues). So if you want to distribute your work load across multiple queues you should
 * tie the frontQueue to a cache such as redis.
 *
 * Default priorities:
 * [
 *  1st position: seed list,
 *  2nd position: updated in last 2 weeks.
 *  never: updated more than two weeks ago.
 * ]
 *
 * Note: we put this description on the FrontQueue type so that it's description will be wherever it is used.
 * */
export declare type FrontQueue = AsyncQueue<Link>[];
/** Links go from the front queue and are enqueued into the back queue using a backQueue router. Each backQueue is nonempty and only
 * one is allowed per host and each queue gets it's own async "thread" (concurrency is what we'll use for the moment instead of
 * parallelization [although concurrency can actually enable parallelization]).
 * This back queue is dequeued from the fetcher. */
export declare type BackQueue = Map<Host, AsyncQueue<Link>>;
/** The Host of a URL */
export declare type Host = string;
/** Tells the time of the last call for an item given the host. */
export declare type HostTimeOfLastCallHeap = AsyncMap<Host, number>;
export declare type URLFrontierSettings = {
    /** This function sends a url from outside the URL Frontier into the frontQueue.
     * This function will tell how a url should be prioritized over others. It should return a number between 1 and F, if it returns undefined the item will be dropped and not be added to the front queue,
     * Whatever number is returned will be clamped between the values of 1 and F. */
    frontQueuePrioritizer(context: {
        url: Link;
        frontQueue: FrontQueue;
    }): Promise<FrontQueue[number]>;
    /** This tells which should be the next URL that comes from the front queue.
     * It MUST NOT handle dequeueing from one of the FrontQueues and returning a URL.
     */
    biasedFrontQueueSelector(context: {
        frontQueue: FrontQueue;
    }): Promise<FrontQueue[number]>;
    /** The number of different priorities allowed in the queue */
    F: number;
    /** The number of different backqueues allowed usually the same as the number of hosts. */
    B: number;
    /** This function is used to assign an item to a backQueue with Politeness guarantees.
     * Should return a member of the backqueue.
     * It MUST NOT handle enqueueing or dequeueing into one of the BackQueues */
    backqueueRouter(context: {
        backQueue: BackQueue;
        host: Host;
    }): Promise<ReturnType<BackQueue["get"]>>;
    /** This function gets something from the back queue */
    backqueueSelector(context: {
        backQueue: BackQueue;
        hostHeap: HostHeap;
        backQueueRouter: URLFrontierSettings["backqueueRouter"];
    }): Promise<Link | undefined>;
};
export declare const defaultURLFrontierSettings: URLFrontierSettings;
export declare class URLFrontier {
    #private;
    settings: URLFrontierSettings;
    constructor(settings?: Partial<URLFrontierSettings>);
    prioritize(url: Link): Promise<Link>;
    seedURL(url: Link): Promise<{
        url: string;
        cached: boolean;
    }>;
    sendURL(url: Link): Promise<{
        url: string;
        cached: boolean;
    }>;
    private _seedURL;
    stop(): void;
    runToCompletion(): Promise<void>;
    [Symbol.asyncIterator](): AsyncGenerator<string, void, unknown>;
    prioritizingFrontQueueAdder(): Promise<void>;
    addToBackQueueFromFrontQueue(): Promise<void>;
    backQueueSelection(): AsyncGenerator<string, void, unknown>;
}
//# sourceMappingURL=url-frontier.d.ts.map