/** * Redis-backed distributed crawl frontier. * * Replaces the in-memory CrawlFrontier for horizontally-scaled crawler deployments. * Multiple crawler workers on different containers can share the same URL queue for * a given job — enabling 10x crawl throughput on large enterprise apps. * * Redis key namespace: zeta:frontier:{jobId} * LIST zeta:frontier:{jobId}:queue — BFS queue (LPUSH/RPOP) * SET zeta:frontier:{jobId}:visited — deduplication set * EXPIRE — 24h TTL on all keys * * Graceful fallback: if REDIS_URL is not set, RedisCrawlFrontier silently falls * back to in-memory arrays so local development works without Redis. */ export declare class RedisCrawlFrontier { private readonly jobId; private redis; private queueKey; private visitedKey; private inMemoryQueue; private inMemoryVisited; private useRedis; constructor(jobId: string); init(): Promise; /** Add a URL to the frontier. No-op if already visited. Returns true if added. */ push(url: string): Promise; /** Pop the next URL (FIFO/BFS). Returns null if queue empty. */ shift(): Promise; /** Check if a URL has already been visited/queued. */ hasVisited(url: string): Promise; /** Current queue length. */ length(): Promise; /** Total visited count. */ visitedCount(): Promise; /** Seed initial URLs (used at job start). */ seed(urls: string[]): Promise; /** Clean up all Redis keys for this frontier. Call at job completion. */ cleanup(): Promise; /** Stats for monitoring. */ stats(): Promise<{ backend: 'redis' | 'memory'; queued: number; visited: number; }>; } /** Check if Redis is configured for distributed frontier. */ export declare function isRedisFrontierEnabled(): boolean;