/** * Headless-browser render tier (parent side). * * Drives the bot-evasion fallback by spawning browser-worker.ts in a DEDICATED, * HARD-KILLABLE child process. The worker (and the Chromium it launches) never * shares memory with the pi agent: only a single JSON result string crosses back. * The child is spawned detached (its own process group) so a wedged renderer can * be reclaimed with one SIGKILL of the whole tree on timeout/abort. * * Playwright is an OPTIONAL dependency — `browserAvailable()` probes for it * without importing/launching anything, so the escalation degrades gracefully * (with an actionable install message) when it is absent. */ import { spawn } from "node:child_process"; import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { EXTENSION_DIR } from "../config.ts"; import { assertSafeUrl } from "./safe-fetch.ts"; const WORKER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), "browser-worker.ts"); /** True when the `playwright` package resolves (does not import or launch it). */ export function browserAvailable(): boolean { try { createRequire(import.meta.url).resolve("playwright"); return true; } catch { return false; } } export function browserDepError(extensionDir?: string): string { const where = extensionDir ? `cd ${extensionDir} && ` : ""; return ( "Headless-browser fallback needs Playwright (optional dependency). Install it:\n" + ` ${where}npm install playwright && npx playwright install chromium` ); } /** * Short user-facing hint appended to block warnings/footers when the missing * fallback would likely have helped. Details live in browserSetupBriefing. */ export const BROWSER_HINT = "the optional headless-browser fallback is not installed — ask pi to enable it (it can fetch most bot-protected pages)"; /** * Agent-facing briefing on the missing optional dependency: why it matters, * how to install it, and the Chromium-free alternative. Injected into the tool * guidelines at startup and into blocked-fetch failures, so the agent can fix * its own setup when the user asks — the user only sees the short BROWSER_HINT. */ export function browserSetupBriefing(): string { return ( "OPTIONAL DEPENDENCY MISSING — the headless-browser fallback (Playwright + Chromium) is not installed, " + "so bot-protected pages (Cloudflare challenges, Reddit, many JS-heavy sites) cannot be fetched. With it " + "installed, blocked fetches escalate automatically to a sandboxed headless Chromium that runs the page's " + "JavaScript and passes most bot checks — more pages read per search, fewer [BLOCKED-DOMAIN] dead ends, " + "richer briefings. To enable it, confirm with the user first (one-time ~150 MB Chromium download), then run:\n" + ` cd ${EXTENSION_DIR} && npm install playwright && npx playwright install chromium\n` + "No config change is needed (browserFallbackEnabled defaults to true; the user can check with " + "/web-research-status). Chromium-free alternative: set readerEndpoint in " + `${path.join(EXTENSION_DIR, "config.json")} to a self-hosted Jina-Reader or FlareSolverr instance.` ); } export interface RenderOptions { timeoutMs: number; allowPrivateNetwork: boolean; maxBytes: number; noSandbox: boolean; userAgent?: string; acceptLanguage?: string; signal?: AbortSignal; } export interface RenderResult { html?: string; blocked?: true; status?: number; blockReason?: string; error?: string; unavailable?: boolean; sandboxFailure?: boolean; } function runWorker(req: string, timeoutMs: number, signal?: AbortSignal): Promise { return new Promise((resolve) => { const child = spawn(process.execPath, ["--no-warnings", WORKER_PATH, req], { detached: true, // own process group → killable as a tree stdio: ["ignore", "pipe", "ignore"], // discard Chromium's stderr noise }); let out = ""; let settled = false; const killTree = () => { try { if (child.pid) process.kill(-child.pid, "SIGKILL"); } catch { /* group may already be gone */ } try { child.kill("SIGKILL"); } catch { /* ignore */ } }; const finish = (r: RenderResult) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener("abort", onAbort); resolve(r); }; // Generous slack over the worker's own timeout to cover spawn + Chromium launch. const timer = setTimeout(() => { killTree(); finish({ error: "browser render timed out" }); }, timeoutMs + 10_000); const onAbort = () => { killTree(); finish({ error: "aborted" }); }; signal?.addEventListener("abort", onAbort, { once: true }); child.stdout?.on("data", (d) => { out += d.toString(); }); child.on("error", (e) => finish({ error: `browser worker failed to start: ${e.message}` })); child.on("exit", () => { const line = out.trim().split("\n").filter(Boolean).pop(); if (!line) return finish({ error: "browser worker produced no output" }); try { finish(JSON.parse(line) as RenderResult); } catch { finish({ error: "browser worker output was not parseable" }); } }); }); } /** * Render `url` in a sandboxed headless Chromium child process and return its DOM * as HTML. The URL is pre-validated with the same SSRF guard as the fetch path * before the browser is touched; the worker re-validates every in-page request. */ export async function renderWithBrowser(url: string, opts: RenderOptions): Promise { try { await assertSafeUrl(url, opts.allowPrivateNetwork); } catch (e) { return { error: e instanceof Error ? e.message : String(e) }; } const req = JSON.stringify({ url, timeoutMs: opts.timeoutMs, allowPrivateNetwork: opts.allowPrivateNetwork, maxBytes: opts.maxBytes, noSandbox: opts.noSandbox, userAgent: opts.userAgent, acceptLanguage: opts.acceptLanguage, }); return runWorker(req, opts.timeoutMs, opts.signal); }