/** * Session-scoped ledger of hosts that actively block automated access. * * When a fetch is blocked by bot protection (Cloudflare challenge, 403/429 bot * walls, etc.) AND every escalation tier (headless browser, reader) also fails, * the host is recorded here. The tools then inject an explicit, machine-readable * warning into their results so the main agent stops re-targeting that domain — * the model only ever sees prose in the tool result (there is no `isError` * channel), so the warning must be unambiguous text it can act on. * * Policy is WARN-ONLY: this ledger never causes a fetch to be skipped. A recorded * host is still attempted on a later call (the challenge may have cleared); the * ledger only drives the warnings/annotations. Module-level session state with a * TTL, mirroring `dedup.ts`; `__resetBlockLedger` clears it for tests. */ // Stable, greppable markers — the machine-readable contract with the model. // Keep these identical across every injection site (all live in tools.ts). export const BLOCKED_MARKER = "[BLOCKED-DOMAIN]"; export const DO_NOT_RETRY_MARKER = "[DO-NOT-RETRY]"; const BLOCK_TTL_MS = 30 * 60_000; // a host counts as "known-blocked" for 30 min export interface BlockInfo { status?: number; reason?: string; lastError: string; } interface BlockRecord extends BlockInfo { count: number; at: number; } const blocked = new Map(); // key: lowercased hostname function hostOf(url: string): string | undefined { try { return new URL(url).hostname.toLowerCase(); } catch { return undefined; } } function pruneBlocked(now: number): void { for (const [host, rec] of blocked) if (now - rec.at >= BLOCK_TTL_MS) blocked.delete(host); } /** Record a terminal block for `url`'s host (called only after all tiers fail). */ export function recordBlock(url: string, info: BlockInfo, now: number = Date.now()): void { const host = hostOf(url); if (!host) return; pruneBlocked(now); const prev = blocked.get(host); blocked.set(host, { ...info, count: (prev?.count ?? 0) + 1, at: now }); } /** Return the live block record for `url`'s host, or undefined if none/expired. */ export function isBlockedHost(url: string, now: number = Date.now()): BlockRecord | undefined { const host = hostOf(url); if (!host) return undefined; pruneBlocked(now); return blocked.get(host); } /** * Canonical post-failure warning for a just-blocked URL, embedding the markers so * the agent reliably stops targeting the host. `tiersTried` notes which fallbacks * were attempted (e.g. "browser-render and reader"). `hint` is an optional * user-facing suggestion appended after the base warning (e.g. a Playwright * install tip when the headless browser isn't available). */ export function blockWarning(url: string, info: BlockInfo, tiersTried?: string, hint?: string): string { const host = hostOf(url) ?? url; const why = info.reason ? `${info.reason}${info.status ? `, HTTP ${info.status}` : ""}` : info.lastError; const tried = tiersTried ? ` The ${tiersTried} fallback${tiersTried.includes("and") ? "s" : ""} also could not retrieve it.` : ""; const hintLine = hint ? ` _💡 ${hint}_` : ""; return ( `${BLOCKED_MARKER} ${host} actively blocks automated access (${why}).${tried} ` + `${DO_NOT_RETRY_MARKER} Do not call web_fetch or web_search against ${host} again this session — ` + `it will keep failing. Get this information from a different source/domain instead.${hintLine}` ); } /** Short inline annotation for a known-blocked URL in a result list. */ export function blockAnnotation(url: string, now: number = Date.now()): string | undefined { const rec = isBlockedHost(url, now); if (!rec) return undefined; const host = hostOf(url) ?? url; return ` ⚠ ${BLOCKED_MARKER}: ${host} blocks fetching — ${DO_NOT_RETRY_MARKER} do not web_fetch it`; } /** Test-only: clear all session block state. */ export function __resetBlockLedger(): void { blocked.clear(); }