import { type Config } from '../config.js'; import { type TlsFetchResult } from './tls-tier.js'; import { type DomainClearance } from '../cache/store.js'; import { BrowserAcquirer } from './browser-acquire.js'; import type { RawFetchResult, BrowserAction, Mode, StageError, ContentCompleteness } from '../types.js'; export interface RouterFetchOptions { renderJs?: 'auto' | 'always' | 'never'; useAuth?: boolean; headers?: Record; screenshot?: boolean; actions?: BrowserAction[]; force_refresh?: boolean; mode?: Mode; /** * Conditional-GET headers. When set, the HTTP path sends them with the * request and a 304 response is returned as RawFetchResult with * statusCode=304 + html=''. Routes that always escalate to Playwright * (renderJs=always, useAuth, actions) ignore these headers. */ conditionalHeaders?: { ifNoneMatch?: string; ifModifiedSince?: string; }; /** Optional abort signal. When provided, in-flight HTTP or browser fetches * will be cancelled when the signal fires. No behavior change — signal is * only plumbed here; enforcement lives in the HTTP client and browser pool. */ signal?: AbortSignal; } export interface HttpClient { fetch(url: string, options?: { headers?: Record; timeoutMs?: number; conditionalHeaders?: { ifNoneMatch?: string; ifModifiedSince?: string; }; signal?: AbortSignal; }): Promise<{ url: string; finalUrl: string; html: string; contentType: string; statusCode: number; headers: Record; rawBuffer?: Buffer; }>; } /** * Options accepted by the browser tier's `fetchWithBrowser`. `stealth` opts a * single fetch into the dedicated anti-bot fingerprint-hardening context path. */ export interface BrowserFetchArgs { headers?: Record; storageStatePath?: string; userDataDir?: string; screenshot?: boolean; actions?: BrowserAction[]; cdpUrl?: string; signal?: AbortSignal; stealth?: boolean; injectedCookies?: Array<{ name: string; value: string; domain: string; path?: string; }>; } export interface BrowserPoolInterface { fetchWithBrowser(url: string, options?: BrowserFetchArgs): Promise; /** Optional pre-launch of the browser engine so a later fetch doesn't pay * cold-start inline. Idempotent + best-effort. Pools that don't implement it * simply skip prewarming. */ warm?(): Promise; } export type HttpFetcher = (url: string, options?: { headers?: Record; timeoutMs?: number; signal?: AbortSignal; }) => Promise<{ url: string; html: string; text: string; }>; export type PlaywrightFetcher = (url: string, options?: { timeoutMs?: number; signal?: AbortSignal; }) => Promise<{ html: string; text: string; completeness: ContentCompleteness; }>; /** * Injectable TLS-impersonation fetcher. Same shape as `tlsFetch` * from tls-tier.ts; left injectable so unit tests can stub without touching * the wreq-js native binary. */ export type TlsFetcher = (url: string, options?: { headers?: Record; timeoutMs?: number; signal?: AbortSignal; }) => Promise; /** * Cheap content-type probe. Resolves true when the URL serves a PDF (by HEAD * content-type or magic-bytes). Injectable so router tests don't hit the * network. Defaults to {@link defaultPdfProbe}. */ export type PdfProbe = (url: string, signal?: AbortSignal) => Promise; /** Pluggable hooks to learning/persistence layer so router tests don't need a DB. */ export interface TlsRoutingPersistence { getPreferTls(domain: string): boolean; recordSuccess(domain: string): void; } /** * Anti-bot clearance store seam (S-A2). The router reads a stored clearance * before a dispatch and purges a dead one after a re-challenge. Injectable so * router unit tests exercise reuse without a DB; defaults to the cache store. * * S-A5 extends the seam with the per-host rate-limit backoff window so the same * injection point covers both clearance reuse and origin-politeness backoff. */ export interface ClearanceStore { get(host: string): DomainClearance | null; clear(host: string): void; /** Epoch ms when the host's rate-limit cooldown ends, or null when none. */ getBackoff(host: string): number | null; /** Record a per-host rate-limit cooldown (epoch ms). */ recordBackoff(host: string, untilEpochMs: number): void; } export interface SmartRouterOptions { httpClient?: HttpClient; browserPool?: BrowserPoolInterface; httpFetcher?: HttpFetcher; playwrightFetcher?: PlaywrightFetcher; /** When provided, overrides the default lazy-loaded wreq backend. */ tlsFetcher?: TlsFetcher; /** Persistence for `prefer_tls_impersonation` learning. */ tlsPersistence?: TlsRoutingPersistence; /** Anti-bot clearance reuse store. Defaults to the cache store. */ clearanceStore?: ClearanceStore; /** Overrides the default HEAD/magic-bytes PDF probe (tests inject a stub). */ pdfProbe?: PdfProbe; /** * Coordinates lazy browser-engine acquisition when the browser tier is * entered on a machine without the browser installed. Injectable so router * tests can control the acquisition outcome without touching the installer. * Defaults to a shared {@link BrowserAcquirer}. */ browserAcquirer?: BrowserAcquirer; /** * Opt-in Tier-B escape-hatch fetchers (challenge-solver + hosted reader). * Injectable so router tests exercise the ladder without the real network * rungs. Defaults to a lazy `import('./escape-hatch.js')` so a default * install never loads the module. */ escapeHatch?: EscapeHatchFetchers; } /** The two escape-hatch rung fetchers, injectable for tests. */ export interface EscapeHatchFetchers { solverFetch: typeof import('./escape-hatch.js').solverFetch; hostedReaderFetch: typeof import('./escape-hatch.js').hostedReaderFetch; } interface DomainStats { failureCount: number; preferPlaywright: boolean; } export declare function looksLikeBinaryDownload(url: string): boolean; /** * True when an HTTP/TLS-tier result is a PDF regardless of URL extension — * either the response advertised `application/pdf`, or the buffered bytes begin * with the PDF magic marker. A PDF response is a completed byte-tier result and * must never be re-routed to the browser (which treats it as a download and * hard-errors "Download is starting"). Extension-independent by design. */ export declare function looksLikePdfResult(result: { contentType?: string; rawBuffer?: Buffer; }): boolean; /** * Cheap content-type probe: a HEAD request that reads only `Content-Type`, with * a bounded ranged-GET magic-bytes fallback when HEAD is unreliable (blocked / * missing header). Resolves true when the URL serves a PDF, false otherwise or * on any error — a probe failure must never block a fetch. Bounded to a short * timeout so it adds minimal latency and can only run when we are already about * to pay a browser cold-start. */ export declare function defaultPdfProbe(url: string, signal?: AbortSignal): Promise; /** * Public predicate over a full URL. True when the URL's host * is in the curated anti-bot/TLS-first set or the operator-supplied * WIGOLO_TLS_DOMAINS list — i.e. the same domains {@link SmartRouter.fetch} * routes through the TLS-impersonation tier first. The search-hydration path * uses this to grant those domains a larger per-URL fetch budget so a working * TLS attempt (~1-5s) is not starved by the small balanced per-URL budget. * Returns false (never throws) for malformed URLs. */ export declare function isAntiBotTlsFirstUrl(url: string, extraDomains: readonly string[]): boolean; /** * Whether a browser-tier fetch should use the dedicated anti-bot * fingerprint-hardening (stealth) context, given the configured mode and * whether THIS browser fetch was reached via an anti-bot / challenge * escalation. * * - 'off' → never harden. * - 'on' → harden every browser fetch. * - 'auto' → harden ONLY an anti-bot / challenge escalation (a bot wall the * lower tiers could not clear). A benign SPA-shell render or an * explicit browser request (render_js:'always' / auth / actions) * is left on the pooled default fingerprint — hardening a benign * page adds cost + a distinct context for no anti-bot benefit. */ export declare function stealthForBrowser(config: Pick, ctx: { antiBotEscalation: boolean; }): boolean; export declare class SmartRouter { private readonly domainMap; private readonly httpClient?; private readonly browserPool?; private readonly httpFetcher; private readonly playwrightFetcher; private readonly tlsFetcher; private readonly tlsPersistence; private readonly pdfProbe; private readonly browserAcquirer; private readonly clearanceStore; private readonly escapeHatchOverride; constructor(httpClient: HttpClient, browserPool: BrowserPoolInterface); constructor(options: SmartRouterOptions); /** * The stored clearance for `host` when it is fresh AND presentable by `tier`, * else null. Purges an EXPIRED entry as a side-effect so a dead cookie is not * carried forward. A UA mismatch (e.g. a Firefox-minted clearance for the * Chromium browser tier) returns null WITHOUT clearing — the entry may still * be valid for another tier. */ private clearanceFor; /** * Merge a reused clearance `Cookie:` header for the header tiers (tls/http) * into `headers` without clobbering caller headers. Returns the original * reference when there is no usable clearance so no allocation happens on the * common path. */ private withClearanceHeader; /** * HTTP-tier dispatch that injects a reused clearance `Cookie:` header for the * URL's host (best-effort cross-tier). Every real HTTP dispatch routes through * here so no call site is left unthreaded. The http-client's redirect follower * strips the Cookie on a cross-host hop, so the cookie never leaks. */ private httpClientFetch; private makeDefaultHttpFetcher; /** * Dispatch an implicit/auto-path fetch that would otherwise go to the browser * tier. A browser treats a PDF response as a download and hard-errors * ("Download is starting"), so — for URLs whose extension didn't already * short-circuit to HTTP — we run a cheap content-type probe first. When the * probe says PDF, the fetch is served by the byte tier instead. The probe * only runs on this browser-bound path, so normal HTML fetches (which are * served HTTP-first) never pay for it. */ private browserOrHttpForBinary; /** * Invoke the browser tier and map a hard bot-protection challenge * (ChallengeBlockedError, thrown by the browser pool's anti-bot fast-fail) * to a structured `blocked_by_challenge` stage error instead of letting it * propagate as an unhandled throw. All other errors propagate unchanged. * Every browser-tier call site routes through here so the mapping is uniform. * * This is ALSO the single lazy-acquisition choke point (D3): before touching * the browser pool we ensure the browser engine is installed. When it is not, * a memoized background install is joined for a bounded budget; if it hasn't * finished in time we DON'T block the tool call for minutes — we fall back to * the best lower-tier content the caller already has (`fallback`, when the * escalation site captured an HTTP/TLS response) with an actionable note, or, * when no lower-tier content exists, return an actionable stage error. */ private browserFetch; /** * Opt-in Tier-B escape-hatch ladder, tried only after the browser tier hits a * hard challenge-block. Rungs are attempted in order: challenge-solver, then * hosted reader. Each is OFF unless its URL is configured; when neither is * configured this returns null WITHOUT loading the escape-hatch module, so a * default install never pays for it. Any rung that clears the page wins. */ private tryEscapeHatch; /** * Terminal choke point: never return a challenge-shell body as final fetch * content. When a lower tier's result is about to exit to the caller as final * content (http-only mode, TLS terminal, escalation exhausted) and the body * classifies as a challenge shell (challenge markers + skeleton, at any HTTP * status), map it to the same `blocked_by_challenge` stage error the browser * tier raises — the caller gets a structured, actionable error instead of the * interstitial markdown. A clean result passes through untouched. */ private guardChallengeShell; fetch(url: string, options: RouterFetchOptions & { mode: 'stealth'; }): Promise; fetch(url: string, options?: RouterFetchOptions): Promise; getDomainStats(domain: string): DomainStats | undefined; /** * Pre-launch the browser engine so a subsequent fetch that escalates to the * browser doesn't pay the cold-start inline. Best-effort and idempotent — * a no-op when no browser pool is configured or the pool doesn't support * warming. Latency-only; never changes fetch results. */ prewarmBrowser(): Promise; private ensureStats; static isKnownSpaDomain(host: string): boolean; private toRawFetchResult; /** * When `host` is inside an active rate-limit backoff window, returns a * synthetic 429 result WITHOUT touching the origin — a 429 is the origin * rate-limiting us, so re-hammering it (even via stealth/force_refresh) does * not help and is impolite. The plain-text body carries no challenge markers, * so it maps to `http_429` downstream exactly as a real pass-through 429 does. * Returns null when there is no active window (null or expired) so the caller * fetches normally — no false positives. */ private backoffWindowResult; /** * Attempt the TLS-impersonation tier for `url`. Returns: * - { ok: true, result } when the tier completed AND the response does * not look like a still-blocking challenge / JS-required page * - { ok: false, reason } when the tier is unavailable (missing native * binary, network error) OR the response still looks anti-bot / JS- * required so the router should escalate to Playwright. * * Records success against the domain on every healthy response. The * `prefer_tls_impersonation` flip is performed by the persistence layer * once the success threshold is reached. */ private tryTlsTier; } export {}; //# sourceMappingURL=router.d.ts.map