import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback, } from "@f5-sales-demo/pi-agent-core"; import { StringEnum } from "@f5-sales-demo/pi-ai"; import { getPuppeteerDir, logger, prompt, Snowflake, untilAborted } from "@f5-sales-demo/pi-utils"; import { Readability } from "@mozilla/readability"; import { type Static, Type } from "@sinclair/typebox"; import { type HTMLElement, parseHTML } from "linkedom"; import type { Browser, CDPSession, ElementHandle, KeyInput, Page, default as Puppeteer, SerializedAXNode, } from "puppeteer"; import { click, fill, resolve, scrollIntoView, waitFor } from "../browser"; import { locateChrome } from "../browser/chrome-locate"; import browserDescription from "../prompts/tools/browser.md" with { type: "text" }; import type { ToolSession } from "../sdk"; import { resizeImage } from "../utils/image-resize"; import { htmlToBasicMarkdown } from "../web/scrapers/types"; import { buildStealthBundle } from "./browser-stealth"; import { deriveUserAgentOverride, type UserAgentOverride } from "./browser-user-agent"; import type { OutputMeta } from "./output-meta"; import { expandPath, resolveToCwd } from "./path-utils"; import { formatScreenshot } from "./render-utils"; import { ToolAbortError, ToolError, throwIfAborted } from "./tool-errors"; import { toolResult } from "./tool-result"; import { clampTimeout } from "./tool-timeouts"; /** * Returns the `browser.connectUrl` setting value if it is a non-empty string, * otherwise undefined. When set, the browser tool will attach to the * already-running Chrome instance instead of launching a new one. */ export function resolveBrowserConnectUrl(settings: { get(key: string): unknown }): string | undefined { const v = settings.get("browser.connectUrl"); return typeof v === "string" && v.trim() ? v.trim() : undefined; } /** * Asserts that a browser connect URL is a loopback http: URL. * Throws with a descriptive message if the URL is invalid, uses a non-http * protocol, or points to a non-loopback address. * * This is intentionally NOT called from `resolveBrowserConnectUrl` (which must * stay pure/non-throwing for use in `#closeBrowser`). Call this guard at the * connect site before `puppeteer.connect`. */ export function assertLoopbackBrowserUrl(url: string): void { let parsed: URL; try { parsed = new URL(url); } catch { throw new Error(`browser.connectUrl is not a valid URL: ${url}`); } if (parsed.protocol !== "http:") throw new Error(`browser.connectUrl must use http: (got ${parsed.protocol})`); const host = parsed.hostname .toLowerCase() .replace(/^\[|\]$/g, "") .replace(/\.+$/, ""); if (host !== "localhost" && host !== "127.0.0.1" && host !== "::1") { throw new Error( `browser.connectUrl must be a loopback address (localhost, 127.0.0.1, or ::1) — got ${parsed.hostname}`, ); } } /** * Returns the index of the best page to reuse when co-driving the human's * Chrome. Scans backwards for the last non-`about:blank` page (i.e. the * active foreground tab); falls back to index 0 when every page is blank. */ export function pickCoDrivePage(pages: { url(): string }[]): number { for (let i = pages.length - 1; i >= 0; i--) { if (pages[i].url() !== "about:blank") return i; } return 0; } /** * Lazy-import puppeteer from a safe CWD so its config loader doesn't choke on * malformed package.json files in the user's project tree. * * Puppeteer 25 swapped cosmiconfig for lilconfig, so the original rationale no * longer names the right library. Retained deliberately rather than removed: * importing puppeteer 25 from a directory containing a deliberately malformed * package.json was verified to succeed, but that only exercises import-time * loading — whether `launch()` reads config lazily was not established, so * dropping the guard would be an unverified behaviour change for a demo-critical * path. It costs one chdir. */ let puppeteerModule: typeof Puppeteer | undefined; async function loadPuppeteer(): Promise { if (puppeteerModule) return puppeteerModule; const prev = process.cwd(); const safeDir = getPuppeteerDir(); await Bun.write(path.join(safeDir, "package.json"), "{}"); try { process.chdir(safeDir); puppeteerModule = (await import("puppeteer")).default; return puppeteerModule; } finally { process.chdir(prev); } } const DEFAULT_VIEWPORT = { width: 1365, height: 768, deviceScaleFactor: 1.25 }; const STEALTH_IGNORE_DEFAULT_ARGS = [ "--disable-extensions", "--disable-default-apps", "--disable-component-extensions-with-background-pages", ]; /** Kept in step with the locale profile in puppeteer/09_stealth_locale.txt. */ const STEALTH_TIMEZONE = "America/New_York"; const PUPPETEER_SOURCE_URL_SUFFIX = "//# sourceURL=__puppeteer_evaluation_script__"; const INTERACTIVE_AX_ROLES = new Set([ "button", "link", "textbox", "combobox", "listbox", "option", "checkbox", "radio", "switch", "tab", "menuitem", "menuitemcheckbox", "menuitemradio", "slider", "spinbutton", "searchbox", "treeitem", ]); /** * Stealth init scripts for Puppeteer. */ type PuppeteerCdpClient = { send: (method: string, params?: Record) => Promise; }; function resolvePageClient(page: Page): PuppeteerCdpClient | null { const pageWithClient = page as Page & { _client?: (() => PuppeteerCdpClient) | PuppeteerCdpClient; }; if (!pageWithClient._client) return null; return typeof pageWithClient._client === "function" ? pageWithClient._client() : pageWithClient._client; } const puppeteerGetArgsSchema = Type.Array( Type.Object({ selector: Type.String({ description: "Selector for the target element (CSS, or puppeteer query handler like aria/, text/, xpath/, pierce/; also accepts legacy p- prefixes)", }), attribute: Type.Optional(Type.String({ description: "Attribute name (get_attribute)" })), }), { description: "Batch arguments for get_* actions", minItems: 1 }, ); const browserSchema = Type.Object({ action: StringEnum( [ "open", "goto", "observe", "click", "click_id", "type", "type_id", "fill", "fill_id", "press", "scroll", "drag", "wait_for_selector", "evaluate", "get_text", "get_html", "get_attribute", "extract_readable", "screenshot", "close", ], { description: "Action to perform" }, ), url: Type.Optional(Type.String({ description: "URL to navigate to (goto)" })), selector: Type.Optional( Type.String({ description: "Selector for the target element (CSS, or puppeteer query handler like aria/, text/, xpath/, pierce/; also accepts legacy p- prefixes)", }), ), element_id: Type.Optional(Type.Number({ description: "Element ID from observe" })), include_all: Type.Optional(Type.Boolean({ description: "Include non-interactive nodes in observe" })), viewport_only: Type.Optional(Type.Boolean({ description: "Limit observe output to elements in the viewport" })), args: Type.Optional(puppeteerGetArgsSchema), script: Type.Optional(Type.String({ description: "JavaScript to evaluate (evaluate)" })), text: Type.Optional(Type.String({ description: "Text to type (type)" })), value: Type.Optional(Type.String({ description: "Value to set (fill)" })), attribute: Type.Optional(Type.String({ description: "Attribute name to read (get_attribute)" })), key: Type.Optional(Type.String({ description: "Keyboard key to press (press)" })), timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (default: 30)" })), wait_until: Type.Optional( StringEnum(["load", "domcontentloaded", "networkidle0", "networkidle2"], { description: "Navigation wait condition (goto)", }), ), full_page: Type.Optional(Type.Boolean({ description: "Capture full page screenshot (screenshot)" })), format: Type.Optional( StringEnum(["text", "markdown"], { description: "Output format for extract_readable (text/markdown)", }), ), path: Type.Optional(Type.String({ description: "Optional path to save screenshot (relative to cwd)" })), viewport: Type.Optional( Type.Object({ width: Type.Number({ description: "Viewport width in pixels" }), height: Type.Number({ description: "Viewport height in pixels" }), device_scale_factor: Type.Optional(Type.Number({ description: "Device scale factor" })), }), ), delta_x: Type.Optional(Type.Number({ description: "Scroll delta X (scroll)" })), delta_y: Type.Optional(Type.Number({ description: "Scroll delta Y (scroll)" })), from_selector: Type.Optional( Type.String({ description: "Drag start selector (CSS, or puppeteer query handler like aria/, text/, xpath/, pierce/; also accepts legacy p- prefixes)", }), ), to_selector: Type.Optional( Type.String({ description: "Drag end selector (CSS, or puppeteer query handler like aria/, text/, xpath/, pierce/; also accepts legacy p- prefixes)", }), ), }); /** Input schema for the Puppeteer tool. */ export type BrowserParams = Static; /** Details describing a Puppeteer tool execution result. */ export interface BrowserToolDetails { action: BrowserParams["action"]; url?: string; selector?: string; elementId?: number; result?: string | string[]; screenshotPath?: string; mimeType?: string; bytes?: number; viewport?: { width: number; height: number; deviceScaleFactor?: number }; observation?: Observation; readable?: ReadableResult; meta?: OutputMeta; } export interface ObservationEntry { id: number; role: string; name?: string; value?: string | number; description?: string; keyshortcuts?: string; states: string[]; } export interface Observation { url: string; title?: string; viewport: { width: number; height: number; deviceScaleFactor?: number }; scroll: { x: number; y: number; width: number; height: number; scrollWidth: number; scrollHeight: number; }; elements: ObservationEntry[]; } export interface ReadableResult { url: string; title?: string; byline?: string; excerpt?: string; contentLength: number; text?: string; markdown?: string; } type ReadableFormat = "text" | "markdown"; /** Trim to non-empty string or undefined. */ function normalize(text: string | null | undefined): string | undefined { const trimmed = text?.trim(); return trimmed || undefined; } /** * Extract readable content from raw HTML. * Tries Readability (article-isolation scoring) first, then falls back to a * CSS selector chain over the same pre-parsed DOM. Returns null if neither * path yields usable content. */ export function extractReadableFromHtml(html: string, url: string, format: ReadableFormat): ReadableResult | null { const { document } = parseHTML(html); // --- Primary: Readability article extraction --- const article = new Readability(document).parse(); if (article) { const result = toReadableResult(url, format, article.textContent, article.content, { title: article.title, byline: article.byline, excerpt: article.excerpt, length: article.length, }); if (result) return result; } // --- Fallback: CSS selector chain --- const candidates = [ document.querySelector("[data-pagefind-body]"), document.querySelector("main article"), document.querySelector("article"), document.querySelector("main"), document.querySelector("[role='main']"), document.body, ]; for (const el of candidates) { if (!el) continue; const innerHTML = el.innerHTML?.trim(); const textContent = el.textContent?.trim(); if (!innerHTML || !textContent) continue; const result = toReadableResult(url, format, textContent, innerHTML, { title: document.title, excerpt: textContent.slice(0, 240), length: textContent.length, }); if (result) return result; } return null; } /** Shared builder for both extraction paths. */ function toReadableResult( url: string, format: ReadableFormat, textContent: string | null | undefined, htmlContent: string | null | undefined, meta: { title?: string | null; byline?: string | null; excerpt?: string | null; length?: number | null }, ): ReadableResult | null { const text = normalize(textContent); const markdown = format === "markdown" ? (normalize(htmlToBasicMarkdown(htmlContent ?? "")) ?? text) : undefined; const normalizedText = format === "text" ? text : undefined; if (!normalizedText && !markdown) return null; return { url, title: normalize(meta.title), byline: normalize(meta.byline), excerpt: normalize(meta.excerpt), contentLength: meta.length ?? text?.length ?? markdown?.length ?? 0, text: normalizedText, markdown, }; } function ensureParam(value: T | undefined, name: string, action: string): T { if (value === undefined || value === null || value === "") { throw new ToolError(`Missing required parameter '${name}' for action '${action}'.`); } return value; } function formatEvaluateResult(value: unknown): string { if (typeof value === "string") return value; if (value === undefined) return "undefined"; try { const serialized = JSON.stringify(value, null, 2); return serialized ?? "undefined"; } catch { return String(value); } } /** * Puppeteer tool for headless browser automation. */ export class BrowserTool implements AgentTool { readonly name = "puppeteer"; readonly label = "Puppeteer"; readonly description: string; readonly parameters = browserSchema; readonly strict = true; #browser: Browser | null = null; #page: Page | null = null; #currentHeadless: boolean | null = null; #browserSession: CDPSession | null = null; #userAgentOverride: UserAgentOverride | null = null; #elementIdCounter = 0; readonly #elementCache = new Map(); readonly #patchedClients = new WeakSet(); constructor(private readonly session: ToolSession) { this.description = prompt.render(browserDescription, {}); } async #closeBrowser(): Promise { await this.#clearElementCache(); const connectUrl = resolveBrowserConnectUrl(this.session.settings); if (connectUrl) { // Attached mode: detach the CDP connection without touching the human's // Chrome instance or their active tab. if (this.#browser?.connected) { this.#browser.disconnect(); } } else { // Launched mode: we own the browser process — close the page then the browser. if (this.#page && !this.#page.isClosed()) { await this.#page.close(); } if (this.#browser?.connected) { await this.#browser.close(); } } this.#page = null; this.#browser = null; this.#browserSession = null; this.#userAgentOverride = null; } async #resetBrowser(params?: BrowserParams): Promise { await this.#closeBrowser(); this.#currentHeadless = this.session.settings.get("browser.headless"); const vp = params?.viewport; const initialViewport = vp ? { width: vp.width, height: vp.height, deviceScaleFactor: vp.device_scale_factor ?? DEFAULT_VIEWPORT.deviceScaleFactor, } : DEFAULT_VIEWPORT; const puppeteer = await loadPuppeteer(); const connectUrl = resolveBrowserConnectUrl(this.session.settings); if (connectUrl) { assertLoopbackBrowserUrl(connectUrl); try { this.#browser = await puppeteer.connect({ browserURL: connectUrl }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); throw new Error( `Could not attach to Chrome at ${connectUrl}: ${msg}. Start Chrome with --remote-debugging-port=9222 and log into your tenant, then retry.`, ); } } else { const launchArgs = [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-blink-features=AutomationControlled", `--window-size=${initialViewport.width},${initialViewport.height}`, ]; const proxy = process.env.PUPPETEER_PROXY; if (proxy) { launchArgs.push(`--proxy-server=${proxy}`); // Chrome (since v72) bypasses proxies for localhost by default. When PUPPETEER_PROXY_BYPASS_LOOPBACK // is true, add <-loopback> so traffic to localhost reaches the proxy (e.g. for mitmdump/auth capture). const bypassLoopback = process.env.PUPPETEER_PROXY_BYPASS_LOOPBACK?.toLowerCase(); if ( bypassLoopback === "true" || bypassLoopback === "1" || bypassLoopback === "yes" || bypassLoopback === "on" ) { launchArgs.push("--proxy-bypass-list=<-loopback>"); } } const ignoreCert = process.env.PUPPETEER_PROXY_IGNORE_CERT_ERRORS?.toLowerCase(); if (ignoreCert === "true" || ignoreCert === "1" || ignoreCert === "yes" || ignoreCert === "on") { launchArgs.push("--ignore-certificate-errors"); } this.#browser = await puppeteer.launch({ headless: this.#currentHeadless, defaultViewport: this.#currentHeadless ? initialViewport : null, executablePath: locateChrome()?.path, args: launchArgs, ignoreDefaultArgs: [...STEALTH_IGNORE_DEFAULT_ARGS], }); } if (connectUrl) { // Co-drive mode: reuse the human's existing foreground tab. const pages = await this.#browser.pages(); this.#page = pages.length ? pages[pickCoDrivePage(pages)] : await this.#browser.newPage(); } else { this.#page = await this.#browser.newPage(); } await this.#applyStealthPatches(this.#page); if (this.#currentHeadless || params?.viewport) { await this.#applyViewport(this.#page, params?.viewport); } return this.#page; } async #ensurePage(params?: BrowserParams): Promise { const desiredHeadless = this.session.settings.get("browser.headless"); if (this.#currentHeadless !== null && this.#currentHeadless !== desiredHeadless) { return this.#resetBrowser(params); } if (this.#page && !this.#page.isClosed()) { return this.#page; } if (!this.#browser?.connected) { return this.#resetBrowser(params); } // co-drive: reuse the human's current tab when attached, else a fresh page const connectUrl = resolveBrowserConnectUrl(this.session.settings); if (connectUrl) { const pages = await this.#browser.pages(); this.#page = pages.length ? pages[pickCoDrivePage(pages)] : await this.#browser.newPage(); } else { this.#page = await this.#browser.newPage(); } await this.#applyStealthPatches(this.#page); if (this.#currentHeadless || params?.viewport) { await this.#applyViewport(this.#page, params?.viewport); } return this.#page; } async #applyViewport(page: Page, viewport?: BrowserParams["viewport"]): Promise { if (!viewport) { await page.setViewport(DEFAULT_VIEWPORT); return; } await page.setViewport({ width: viewport.width, height: viewport.height, deviceScaleFactor: viewport.device_scale_factor ?? DEFAULT_VIEWPORT.deviceScaleFactor, }); } async #clearElementCache(): Promise { if (this.#elementCache.size === 0) { this.#elementIdCounter = 0; return; } const handles = Array.from(this.#elementCache.values()); this.#elementCache.clear(); this.#elementIdCounter = 0; await Promise.all( handles.map(async handle => { try { await handle.dispose(); } catch { return; } }), ); } async #resolveCachedHandle(id: number): Promise { const handle = this.#elementCache.get(id); if (!handle) { throw new ToolError(`Unknown element_id ${id}. Run observe to refresh the element list.`); } try { const isConnected = (await handle.evaluate(el => el.isConnected)) as boolean; if (!isConnected) { await this.#clearElementCache(); throw new ToolError(`Element_id ${id} is stale. Run observe again.`); } } catch { await this.#clearElementCache(); throw new ToolError(`Element_id ${id} is stale. Run observe again.`); } return handle; } #isInteractiveNode(node: SerializedAXNode): boolean { if (INTERACTIVE_AX_ROLES.has(node.role)) return true; return ( node.checked !== undefined || node.pressed !== undefined || node.selected !== undefined || node.expanded !== undefined || node.focused === true ); } async #collectObservationEntries( node: SerializedAXNode, entries: ObservationEntry[], options: { viewportOnly: boolean; includeAll: boolean }, ): Promise { if (options.includeAll || this.#isInteractiveNode(node)) { const handle = await node.elementHandle(); if (handle) { let inViewport = true; if (options.viewportOnly) { try { inViewport = await handle.isIntersectingViewport(); } catch { inViewport = false; } } if (inViewport) { const id = ++this.#elementIdCounter; const states: string[] = []; if (node.disabled) states.push("disabled"); if (node.checked !== undefined) states.push(`checked=${String(node.checked)}`); if (node.pressed !== undefined) states.push(`pressed=${String(node.pressed)}`); if (node.selected !== undefined) states.push(`selected=${String(node.selected)}`); if (node.expanded !== undefined) states.push(`expanded=${String(node.expanded)}`); if (node.required) states.push("required"); if (node.readonly) states.push("readonly"); if (node.multiselectable) states.push("multiselectable"); if (node.multiline) states.push("multiline"); if (node.modal) states.push("modal"); if (node.focused) states.push("focused"); this.#elementCache.set(id, handle); entries.push({ id, role: node.role, name: node.name, value: node.value, description: node.description, keyshortcuts: node.keyshortcuts, states, }); } else { await handle.dispose(); } } } for (const child of node.children ?? []) { await this.#collectObservationEntries(child, entries, options); } } #formatObservation(observation: Observation): string { const viewport = `${observation.viewport.width}x${observation.viewport.height}`; const scroll = `x=${observation.scroll.x} y=${observation.scroll.y} viewport=${observation.scroll.width}x${observation.scroll.height} doc=${observation.scroll.scrollWidth}x${observation.scroll.scrollHeight}`; const lines = [ `URL: ${observation.url}`, observation.title ? `Title: ${observation.title}` : "Title:", `Viewport: ${viewport}`, `Scroll: ${scroll}`, "Elements:", ]; for (const entry of observation.elements) { const name = entry.name ? ` "${entry.name}"` : ""; const value = entry.value !== undefined ? ` value=${JSON.stringify(entry.value)}` : ""; const description = entry.description ? ` desc=${JSON.stringify(entry.description)}` : ""; const shortcuts = entry.keyshortcuts ? ` shortcuts=${JSON.stringify(entry.keyshortcuts)}` : ""; const state = entry.states.length ? ` (${entry.states.join(", ")})` : ""; lines.push(`${entry.id}. ${entry.role}${name}${value}${description}${shortcuts}${state}`); } return lines.join("\n"); } /** * Restart the browser to apply changes like headless mode. */ async restartForModeChange(): Promise { await this.#resetBrowser(); } async #applyStealthPatches(page: Page): Promise { this.#patchSourceUrl(page); await this.#applyUserAgentOverride(page); await this.#applyTimezoneOverride(page); await this.#injectStealthScripts(page); } /** * Overrides the timezone through CDP rather than in the injected scripts. * * Chrome recomputes Date, Intl, getTimezoneOffset and the zone name from the * overridden zone, so they all agree. Patching them in page script cannot * achieve that: the previous attempt rewrote only the displayed zone NAME and * forced its zone into every Intl.DateTimeFormat call, which overrode a * caller's explicit `timeZone` and corrupted formatted output (see * 09_stealth_locale.txt). */ async #applyTimezoneOverride(page: Page): Promise { const client = resolvePageClient(page); if (!client) return; try { await client.send("Emulation.setTimezoneOverride", { timezoneId: STEALTH_TIMEZONE }); } catch (error) { // A rejected zone id must not take the rest of the stealth setup down. logger.debug("Failed to apply timezone override", { error: error instanceof Error ? error.message : String(error), }); } } async #applyUserAgentOverride(page: Page): Promise { const client = resolvePageClient(page); if (!client) return; const override = await this.#resolveUserAgentOverride(page); await this.#sendUserAgentOverride(client, override); await this.#configureUserAgentTargets(override); } async #resolveUserAgentOverride(page: Page): Promise { if (this.#userAgentOverride) return this.#userAgentOverride; const browser = page.browser(); this.#userAgentOverride = deriveUserAgentOverride(await browser.userAgent(), await browser.version()); return this.#userAgentOverride; } async #configureUserAgentTargets(override: UserAgentOverride): Promise { if (!this.#browser) return; if (!this.#browserSession) { this.#browserSession = await this.#browser.target().createCDPSession(); await this.#browserSession.send("Target.setAutoAttach", { autoAttach: true, waitForDebuggerOnStart: false, flatten: true, }); this.#browserSession.on("Target.attachedToTarget", async (event: { sessionId: string }) => { const connection = this.#browserSession?.connection(); const session = connection?.session(event.sessionId); if (!session || !this.#userAgentOverride) return; await this.#sendUserAgentOverride(this.#wrapSession(session), this.#userAgentOverride); }); } const targets = this.#browser.targets(); await Promise.all( targets.map(async target => { const session = await target.createCDPSession(); await this.#sendUserAgentOverride(this.#wrapSession(session), override); }), ); } #wrapSession(session: CDPSession): PuppeteerCdpClient { return { send: async (method, params) => session.send(method as never, params as never), }; } async #sendUserAgentOverride(client: PuppeteerCdpClient, override: UserAgentOverride): Promise { try { await client.send("Network.enable"); } catch {} try { await client.send("Network.setUserAgentOverride", override); } catch (error) { logger.debug("Failed to apply Network user agent override", { error: error instanceof Error ? error.message : String(error), }); } try { await client.send("Emulation.setUserAgentOverride", override); } catch (error) { logger.debug("Failed to apply Emulation user agent override", { error: error instanceof Error ? error.message : String(error), }); } } #patchSourceUrl(page: Page): void { const client = resolvePageClient(page); if (!client) return; const clientKey = client as object; if (this.#patchedClients.has(clientKey)) return; this.#patchedClients.add(clientKey); const originalSend = client.send.bind(client); client.send = async (method: string, params?: Record) => { const next = async (payload?: Record) => { try { return await originalSend(method, payload); } catch (error) { if ( error instanceof Error && error.message.includes( "Protocol error (Network.getResponseBody): No resource with given identifier found", ) ) { return undefined; } throw error; } }; if (!method || !params) { return next(params); } const key = method === "Runtime.evaluate" ? "expression" : method === "Runtime.callFunctionOn" ? "functionDeclaration" : null; if (!key) { return next(params); } const value = params[key]; if (typeof value !== "string" || !value.includes(PUPPETEER_SOURCE_URL_SUFFIX)) { return next(params); } const patchedParams = { ...params, [key]: value.replace(PUPPETEER_SOURCE_URL_SUFFIX, "") }; return next(patchedParams); }; } /** Injects stealth scripts that cover common puppeteer detection surfaces. */ async #injectStealthScripts(page: Page): Promise { await page.evaluateOnNewDocument(buildStealthBundle()); } async execute( _toolCallId: string, params: BrowserParams, signal?: AbortSignal, _onUpdate?: AgentToolUpdateCallback, _ctx?: AgentToolContext, ): Promise> { try { throwIfAborted(signal); const timeoutSeconds = clampTimeout("browser", params.timeout); const timeoutMs = timeoutSeconds * 1000; const details: BrowserToolDetails = { action: params.action }; switch (params.action) { case "open": { const page = await untilAborted(signal, () => this.#resetBrowser(params)); const viewport = page.viewport(); details.viewport = viewport ?? DEFAULT_VIEWPORT; const mode = this.#currentHeadless ? "headless" : "visible"; return toolResult(details).text(`Opened browser session (${mode})`).done(); } case "close": { await untilAborted(signal, () => this.#closeBrowser()); return toolResult(details).text("Closed browser session").done(); } case "goto": { const url = ensureParam(params.url, "url", params.action); details.url = url; const page = await this.#ensurePage(params); const waitUntil = params.wait_until ?? "networkidle2"; await this.#clearElementCache(); await untilAborted(signal, () => page.goto(url, { waitUntil, timeout: timeoutMs })); const finalUrl = page.url(); const title = (await untilAborted(signal, () => page.title())) as string; details.url = finalUrl; details.result = title; return toolResult(details) .text(`Navigated to ${finalUrl}${title ? `\nTitle: ${title}` : ""}`) .done(); } case "observe": { const page = await this.#ensurePage(params); const timeoutSignal = AbortSignal.timeout(timeoutMs); const observeSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; await this.#clearElementCache(); const snapshot = (await untilAborted(observeSignal, () => page.accessibility.snapshot({ interestingOnly: !(params.include_all ?? false) }), )) as SerializedAXNode | null; if (!snapshot) { throw new ToolError("Accessibility snapshot unavailable"); } const entries: ObservationEntry[] = []; await this.#collectObservationEntries(snapshot, entries, { viewportOnly: params.viewport_only ?? false, includeAll: params.include_all ?? false, }); const scroll = (await untilAborted(observeSignal, () => page.evaluate(() => { const win = globalThis as unknown as { scrollX: number; scrollY: number; innerWidth: number; innerHeight: number; document: { documentElement: { scrollWidth: number; scrollHeight: number } }; }; const doc = win.document.documentElement; return { x: win.scrollX, y: win.scrollY, width: win.innerWidth, height: win.innerHeight, scrollWidth: doc.scrollWidth, scrollHeight: doc.scrollHeight, }; }), )) as Observation["scroll"]; const url = page.url(); const title = (await untilAborted(observeSignal, () => page.title())) as string; const viewport = page.viewport() ?? DEFAULT_VIEWPORT; const observation: Observation = { url, title, viewport, scroll, elements: entries, }; details.url = url; details.viewport = viewport; details.observation = observation; details.result = `${entries.length} elements`; return toolResult(details).text(this.#formatObservation(observation)).done(); } case "click": { const selector = ensureParam(params.selector, "selector", params.action); details.selector = selector; const page = await this.#ensurePage(params); await untilAborted(signal, () => click(page, selector)); return toolResult(details).text(`Clicked ${selector}`).done(); } case "click_id": { const elementId = ensureParam(params.element_id, "element_id", params.action); details.elementId = elementId; const handle = await this.#resolveCachedHandle(elementId); try { await untilAborted(signal, () => handle.click()); } catch { await this.#clearElementCache(); throw new ToolError(`Element_id ${elementId} is stale. Run observe again.`); } return toolResult(details).text(`Clicked element ${elementId}`).done(); } case "type": { const selector = ensureParam(params.selector, "selector", params.action); const text = ensureParam(params.text, "text", params.action); details.selector = selector; const page = await this.#ensurePage(params); const h = await untilAborted(signal, () => resolve(page, selector)); await untilAborted(signal, () => h.focus()); await untilAborted(signal, () => page.keyboard.type(text, { delay: 0 })); await h.dispose(); return toolResult(details).text(`Typed into ${selector}`).done(); } case "type_id": { const elementId = ensureParam(params.element_id, "element_id", params.action); const text = ensureParam(params.text, "text", params.action); details.elementId = elementId; const page = await this.#ensurePage(params); const handle = await this.#resolveCachedHandle(elementId); try { await untilAborted(signal, () => handle.focus()); await untilAborted(signal, () => page.keyboard.type(text, { delay: 0 })); } catch { await this.#clearElementCache(); throw new ToolError(`Element_id ${elementId} is stale. Run observe again.`); } return toolResult(details).text(`Typed into element ${elementId}`).done(); } case "fill": { const selector = ensureParam(params.selector, "selector", params.action); const value = ensureParam(params.value, "value", params.action); details.selector = selector; const page = await this.#ensurePage(params); await untilAborted(signal, () => fill(page, selector, value)); return toolResult(details).text(`Filled ${selector}`).done(); } case "fill_id": { const elementId = ensureParam(params.element_id, "element_id", params.action); const value = ensureParam(params.value, "value", params.action); details.elementId = elementId; const handle = await this.#resolveCachedHandle(elementId); try { await untilAborted(signal, () => handle.evaluate((el, inputValue) => { const element = el as { value?: string; dispatchEvent: (event: Event) => boolean }; if (!("value" in element)) { throw new Error("Target element is not a form input"); } element.value = String(inputValue); element.dispatchEvent(new Event("input", { bubbles: true })); element.dispatchEvent(new Event("change", { bubbles: true })); }, value), ); } catch { await this.#clearElementCache(); throw new ToolError(`Element_id ${elementId} is stale. Run observe again.`); } return toolResult(details).text(`Filled element ${elementId}`).done(); } case "press": { const key = ensureParam(params.key, "key", params.action) as KeyInput; const page = await this.#ensurePage(params); if (params.selector) { const h = await resolve(page, params.selector as string); await untilAborted(signal, () => h.focus()); await h.dispose(); } await untilAborted(signal, () => page.keyboard.press(key)); return toolResult(details).text(`Pressed ${key}`).done(); } case "scroll": { const page = await this.#ensurePage(params); if (params.selector) { await untilAborted(signal, () => scrollIntoView(page, params.selector as string)); return toolResult(details).text(`Scrolled ${params.selector} into view`).done(); } const deltaY = ensureParam(params.delta_y, "delta_y", params.action); const deltaX = params.delta_x ?? 0; await untilAborted(signal, () => page.mouse.wheel({ deltaX, deltaY })); return toolResult(details).text(`Scrolled by ${deltaX}, ${deltaY}`).done(); } case "drag": { const fromSelector = ensureParam(params.from_selector, "from_selector", params.action); const toSelector = ensureParam(params.to_selector, "to_selector", params.action); const page = await this.#ensurePage(params); const fromHandle = await untilAborted(signal, () => resolve(page, fromSelector)); const toHandle = await untilAborted(signal, () => resolve(page, toSelector)); if (!fromHandle || !toHandle) { throw new ToolError("Drag selectors did not resolve to elements"); } const fromBox = (await untilAborted(signal, () => fromHandle.boundingBox())) as { x: number; y: number; width: number; height: number; } | null; const toBox = (await untilAborted(signal, () => toHandle.boundingBox())) as { x: number; y: number; width: number; height: number; } | null; await fromHandle.dispose(); await toHandle.dispose(); if (!fromBox || !toBox) { throw new ToolError("Drag elements are not visible"); } const startX = fromBox.x + fromBox.width / 2; const startY = fromBox.y + fromBox.height / 2; const endX = toBox.x + toBox.width / 2; const endY = toBox.y + toBox.height / 2; await untilAborted(signal, () => page.mouse.move(startX, startY)); await untilAborted(signal, () => page.mouse.down()); await untilAborted(signal, () => page.mouse.move(endX, endY, { steps: 12 })); await untilAborted(signal, () => page.mouse.up()); return toolResult(details).text(`Dragged from ${fromSelector} to ${toSelector}`).done(); } case "wait_for_selector": { const selector = ensureParam(params.selector, "selector", params.action); details.selector = selector; const page = await this.#ensurePage(params); await untilAborted(signal, () => waitFor(page, selector, undefined, timeoutMs)); return toolResult(details).text(`Selector ready: ${selector}`).done(); } case "evaluate": { const script = ensureParam(params.script, "script", params.action); const page = await this.#ensurePage(params); const value = (await untilAborted(signal, () => page.evaluate(async (source: string) => { try { return await new Function(`return (async () => (${source}))();`)(); } catch { return await new Function(`return (async () => { ${source} })();`)(); } }, script), )) as unknown; const output = formatEvaluateResult(value); details.result = output; return toolResult(details).text(output).done(); } case "get_text": { const page = await this.#ensurePage(params); if (params.args?.length) { const values = (await Promise.all( params.args.map(async (arg, index) => { const selector = ensureParam(arg.selector, `args[${index}].selector`, params.action); const h = await untilAborted(signal, () => resolve(page, selector)); const t = (await h.evaluate( el => (el as unknown as { innerText: string }).innerText, )) as string; await h.dispose(); return t; }), )) as string[]; details.result = values; return toolResult(details) .text(JSON.stringify(values, null, 2)) .done(); } const selector = ensureParam(params.selector, "selector", params.action); details.selector = selector; const h = await untilAborted(signal, () => resolve(page, selector)); const value = (await h.evaluate(el => (el as unknown as { innerText: string }).innerText)) as string; await h.dispose(); details.result = value; return toolResult(details).text(value).done(); } case "get_html": { const page = await this.#ensurePage(params); if (params.args?.length) { const values = (await Promise.all( params.args.map(async (arg, index) => { const selector = ensureParam(arg.selector, `args[${index}].selector`, params.action); const h = await untilAborted(signal, () => resolve(page, selector)); const t = (await h.evaluate( el => (el as unknown as { innerHTML: string }).innerHTML, )) as string; await h.dispose(); return t; }), )) as string[]; details.result = values; return toolResult(details) .text(JSON.stringify(values, null, 2)) .done(); } const selector = ensureParam(params.selector, "selector", params.action); details.selector = selector; const h = await untilAborted(signal, () => resolve(page, selector)); const value = (await h.evaluate(el => (el as unknown as { innerHTML: string }).innerHTML)) as string; await h.dispose(); details.result = value; return toolResult(details).text(value).done(); } case "get_attribute": { const page = await this.#ensurePage(params); if (params.args?.length) { const values = (await Promise.all( params.args.map(async (arg, index) => { const selector = ensureParam(arg.selector, `args[${index}].selector`, params.action); const attr = ensureParam(arg.attribute, `args[${index}].attribute`, params.action); const h = await resolve(page, selector); const v = await h.evaluate((el, a) => (el as HTMLElement).getAttribute(a), attr); await h.dispose(); return v; }), )) as string[]; details.result = values; return toolResult(details) .text(JSON.stringify(values, null, 2)) .done(); } const selector = ensureParam(params.selector, "selector", params.action); const attribute = ensureParam(params.attribute, "attribute", params.action); details.selector = selector; const h = await resolve(page, selector); const value = (await h.evaluate((el, a) => (el as HTMLElement).getAttribute(a), attribute)) as | string | null; await h.dispose(); const output = value ?? ""; details.result = output; return toolResult(details).text(output).done(); } case "extract_readable": { const page = await this.#ensurePage(params); const format = params.format ?? "markdown"; const html = (await untilAborted(signal, () => page.content())) as string; const url = page.url(); const readable = extractReadableFromHtml(html, url, format); if (!readable) { throw new ToolError("Readable content not found"); } details.url = url; details.readable = readable; details.result = format === "markdown" ? (readable.markdown ?? "") : (readable.text ?? ""); return toolResult(details) .text(JSON.stringify(readable, null, 2)) .done(); } case "screenshot": { const page = await this.#ensurePage(params); const fullPage = params.selector ? false : (params.full_page ?? false); let buffer: Buffer; if (params.selector) { const handle = await untilAborted(signal, () => resolve(page, params.selector as string)); buffer = (await untilAborted(signal, () => handle.screenshot({ type: "png" }))) as Buffer; await handle.dispose(); details.selector = params.selector; } else { buffer = (await untilAborted(signal, () => page.screenshot({ type: "png", fullPage }))) as Buffer; } // Compress aggressively for API content — screenshots are the most // frequent image source and land directly in the next LLM request. // 1024px is plenty for OCR/UI inspection; 150KB keeps payloads lean. const resized = await resizeImage( { type: "image", data: buffer.toBase64(), mimeType: "image/png" }, { maxWidth: 1024, maxHeight: 1024, maxBytes: 150 * 1024, jpegQuality: 70 }, ); // Resolve destination: user-defined path > screenshotDir (auto-named) > temp file. const screenshotDir = (() => { const v = this.session.settings.get("browser.screenshotDir") as string | undefined; return v ? expandPath(v) : undefined; })(); const paramPath = params.path ? resolveToCwd(params.path as string, this.session.cwd) : undefined; let dest: string; if (paramPath) { dest = paramPath; } else if (screenshotDir) { const ts = new Date().toISOString().replace(/[:.]/g, "-").slice(0, -1); dest = path.join(screenshotDir, `screenshot-${ts}.png`); } else { dest = path.join(os.tmpdir(), `xcsh-sshots-${Snowflake.next()}.png`); } await fs.promises.mkdir(path.dirname(dest), { recursive: true }); // Full-res buffer when saving to a user-defined location; resized (API copy) for temp-only. const saveFullRes = !!(paramPath || screenshotDir); const savedBuffer = saveFullRes ? buffer : resized.buffer; const savedMimeType = saveFullRes ? "image/png" : resized.mimeType; await Bun.write(dest, savedBuffer); details.screenshotPath = dest; details.mimeType = savedMimeType; details.bytes = savedBuffer.length; const lines = formatScreenshot({ saveFullRes, savedMimeType, savedByteLength: savedBuffer.length, dest, resized, }); return toolResult(details) .content([ { type: "text", text: lines.join("\n") }, { type: "image", data: resized.data, mimeType: resized.mimeType }, ]) .done(); } default: throw new ToolError(`Unsupported action: ${params.action}`); } } catch (error) { if (error instanceof ToolAbortError) throw error; if (error instanceof Error && error.name === "AbortError") { throw new ToolAbortError(); } throw error; } } }