export interface RobotsRule {
    agent: string;
    allow: boolean;
    path: string;
}
export interface ParsedRobots {
    rules: RobotsRule[];
    crawlDelaySec?: number;
}
export type RobotsCheckResult = "allow" | "deny" | "unknown";
export interface RobotsCache {
    get(origin: string): ParsedRobots | null | undefined;
    set(origin: string, parsed: ParsedRobots | null): void;
}
export interface CanFetchOptions {
    userAgent: string;
    cache?: RobotsCache;
    fetchImpl?: typeof fetch;
    timeoutMs?: number;
    signal?: AbortSignal;
}
export declare const DEFAULT_USER_AGENT = "deepdive-bot";
export declare function createRobotsCache(): RobotsCache;
export declare function canFetch(url: string, opts: CanFetchOptions): Promise<RobotsCheckResult>;
export declare function parseRobotsTxt(text: string): ParsedRobots;
export declare function isPathAllowed(parsed: ParsedRobots, path: string, userAgent: string): boolean;
//# sourceMappingURL=robots.d.ts.map