/** * Browser Automation via Chrome DevTools Protocol (CDP) * * Drives a real browser (Edge/Chrome) through the DevTools Protocol by * connecting to a remote debugging port. Zero external dependencies: uses * Node's global WebSocket. * * Registers the `browser` tool with an `action` switch covering: * launch, list, navigate, click, type, js, read, wait, screenshot, close. * * The tool auto-starts a dedicated Edge debugging instance on port 9222 with * an isolated user-data-dir so it does not disturb your normal browsing. * Reuse an already-running 9222 instance when present. * * Note: handshake/verification (CAPTCHA, two-factor) cannot be automated here. */ import type { AgentToolResult, ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; import { StringEnum } from "@earendil-works/pi-ai"; import { execFile } from "node:child_process"; import { existsSync, writeFileSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; const execFileP = promisify(execFile); const DEBUG_PORT = 9222; const EDGE_PATHS = [ "C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe", "C:/Program Files/Microsoft/Edge/Application/msedge.exe", ]; const CHROME_PATHS = [ "C:/Program Files/Google/Chrome/Application/chrome.exe", "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe", ]; function findBrowser(): string | null { for (const p of [...EDGE_PATHS, ...CHROME_PATHS]) { if (existsSync(p)) return p; } return null; } async function httpJson(url: string): Promise { const res = await fetch(url); if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`); return res.json(); } async function isDebugRunning(): Promise { try { await httpJson(`http://localhost:${DEBUG_PORT}/json/version`); return true; } catch { return false; } } function launchBrowser(): Promise { const browser = findBrowser(); if (!browser) throw new Error("No Edge/Chrome found."); const profile = join(homedir(), ".browser-automation-profile"); const child = execFile( browser, [ `--remote-debugging-port=${DEBUG_PORT}`, `--user-data-dir=${profile}`, "--no-first-run", "--no-default-browser-check", "about:blank", ], { detached: true, windowsHide: false } as any, (err: any) => { if ( err && err.code !== "ERR_HTTP_HEADERS_SENT" && err.code !== "ECONNREFUSED" ) { // child detached; errors here are non-fatal for already-running instances } }, ); child.unref(); return Promise.resolve(); } async function waitForDebug(timeoutMs = 8000): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { if (await isDebugRunning()) return; await new Promise((r) => setTimeout(r, 250)); } throw new Error("Browser debugging port did not come up in time."); } /** Minimal CDP client over a page WebSocket. */ class CDP { private ws: WebSocket; private id = 0; private pending = new Map void; reject: (e: Error) => void }>(); private events = new Map void>(); constructor(wsUrl: string) { this.ws = new WebSocket(wsUrl); } async connect(): Promise { this.ws.onmessage = (e) => { const d = JSON.parse(e.data as string); if (d.id && this.pending.has(d.id)) { const p = this.pending.get(d.id)!; this.pending.delete(d.id); if (d.error) p.reject(new Error(`${d.error.code}: ${d.error.message}`)); else p.resolve(d.result); } else if (d.method && this.events.has(d.method)) { this.events.get(d.method)!(d.params); } }; await new Promise((resolve, reject) => { this.ws.onopen = resolve; this.ws.onerror = () => reject(new Error("WebSocket connect failed")); }); } send(method: string, params: any = {}): Promise { return new Promise((resolve, reject) => { const i = ++this.id; this.pending.set(i, { resolve, reject }); this.ws.send(JSON.stringify({ id: i, method, params })); }); } on(method: string, fn: (params: any) => void): void { this.events.set(method, fn); } close(): void { try { this.ws.close(); } catch { /* ignore */ } } } interface CdpManager { cdp: CDP; targetId: string; } // Kept per-session; created lazily and persisted across tool calls. let manager: CdpManager | null = null; async function getManager(pi: ExtensionAPI, action: string): Promise { if (manager) { // Verify still alive try { await httpJson(`http://localhost:${DEBUG_PORT}/json/version`); return manager; } catch { manager = null; } } if (action !== "launch" && !(await isDebugRunning())) { throw new Error( `No debugging browser running on port ${DEBUG_PORT}. Call browser with action="launch" first, or the tool will not auto-start.`, ); } // Pick a page target (prefer real page over background_page/extensions) const targets = await httpJson(`http://localhost:${DEBUG_PORT}/json`); let page = targets.find((t: any) => t.type === "page" && !t.url.startsWith("chrome-extension://")); if (!page) page = targets.find((t: any) => t.type === "page"); if (!page) throw new Error("No page target available."); const cdp = new CDP(page.webSocketDebuggerUrl); await cdp.connect(); await cdp.send("Page.enable"); await cdp.send("Runtime.enable"); manager = { cdp, targetId: page.id }; return manager; } async function evalJs(cdp: CDP, expression: string): Promise { const r = await cdp.send("Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true, }); if (r.exceptionDetails) { throw new Error(`JS error: ${r.exceptionDetails.exception?.description || r.exceptionDetails.text}`); } return r.result?.value; } function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } // ---------- Tool schema ---------- const Action = StringEnum( ["launch", "list", "navigate", "click", "type", "js", "read", "wait", "screenshot", "close"] as const, { description: "Action to perform: launch (start/ensure debug browser), list (list tabs), navigate (go to url), click (CSS selector), type (fill text in CSS selector), js (run arbitrary JS), read (read page/title/sel), wait (wait for CSS selector), screenshot (capture page png), close (close socket).", }, ); const BrowserParams = Type.Object({ action: Action, url: Type.Optional(Type.String({ description: "URL for navigate action" })), selector: Type.Optional(Type.String({ description: "CSS selector for click/type/read/wait" })), text: Type.Optional(Type.String({ description: "Text to type for type action" })), expression: Type.Optional(Type.String({ description: "JS expression for js action" })), timeout: Type.Optional(Type.Integer({ description: "Timeout ms for wait action (default 5000)" })), }); type BrowserDetails = { ok?: boolean; count?: number; action?: string }; export function registerBrowser(pi: ExtensionAPI) { pi.registerTool({ name: "browser", label: "Browser (CDP)", description: "Control a real browser via the Chrome DevTools Protocol. Actions: launch, list, navigate, click (CSS selector), type (fill selector), js (run JS), read (page/title/sel), wait (CSS selector appear), screenshot, close. Uses a dedicated Edge debugging instance on port 9222 that auto-starts on launch.", promptSnippet: "Drive a real browser: navigate, click, type, read, screenshot", parameters: BrowserParams, async execute(_toolCallId, params, signal): Promise> { if (signal?.aborted) return { content: [{ type: "text", text: "Cancelled" }], details: {} }; const action = params.action; if (action === "launch") { if (!(await isDebugRunning())) { launchBrowser(); await waitForDebug(); } const mgr = await getManager(pi, "launch"); const version = await httpJson(`http://localhost:${DEBUG_PORT}/json/version`); return { content: [ { type: "text", text: `Browser ready. Version: ${version.Browser}. Use action="navigate" to go somewhere.`, }, ], details: { ok: true }, }; } if (action === "list") { const targets = await httpJson(`http://localhost:${DEBUG_PORT}/json`); const pages = targets .filter((t: any) => t.type === "page") .map((t: any) => `- ${t.title || "(untitled)"} : ${t.url}`); return { content: [{ type: "text", text: pages.join("\n") || "(no tabs)" }], details: { count: pages.length }, }; } const mgr = await getManager(pi, action); const cdp = mgr.cdp; const out: string[] = []; switch (action) { case "navigate": { if (!params.url) throw new Error("navigate requires url"); await cdp.send("Page.navigate", { url: params.url }); await sleep(800); const title = await evalJs(cdp, "document.title"); out.push(`Navigated to ${params.url}`); out.push(`Title: ${title || "(none)"}`); break; } case "click": { if (!params.selector) throw new Error("click requires selector"); const rect = await evalJs( cdp, `(()=>{const el=document.querySelector(${JSON.stringify(params.selector)});if(!el)return null;const r=el.getBoundingClientRect();return {x:r.x+r.width/2,y:r.y+r.height/2,tag:el.tagName,text:(el.textContent||'').trim().slice(0,80)}})()`, ); if (!rect) throw new Error(`Element not found: ${params.selector}`); await cdp.send("Input.dispatchMouseEvent", { type: "mouseMoved", x: rect.x, y: rect.y }); await cdp.send("Input.dispatchMouseEvent", { type: "mousePressed", x: rect.x, y: rect.y, button: "left", clickCount: 1, }); await cdp.send("Input.dispatchMouseEvent", { type: "mouseReleased", x: rect.x, y: rect.y, button: "left", clickCount: 1, }); await sleep(300); out.push(`Clicked ${params.selector} (${rect.tag})`); if (rect.text) out.push(`text: ${rect.text}`); break; } case "type": { if (!params.selector) throw new Error("type requires selector"); if (params.text === undefined) throw new Error("type requires text"); await evalJs( cdp, `(()=>{const el=document.querySelector(${JSON.stringify(params.selector)});if(!el)throw new Error('not found');el.focus();el.value=${JSON.stringify(params.text)};el.dispatchEvent(new Event('input',{bubbles:true}));el.dispatchEvent(new Event('change',{bubbles:true}));return true})()`, ); const val = await evalJs( cdp, `document.querySelector(${JSON.stringify(params.selector)}).value`, ); out.push(`Typed into ${params.selector}: current value = ${JSON.stringify(val)}`); break; } case "js": { if (!params.expression) throw new Error("js requires expression"); const value = await evalJs(cdp, params.expression); out.push(typeof value === "string" ? value : JSON.stringify(value)); break; } case "read": { if (params.selector) { const text = await evalJs( cdp, `(document.querySelector(${JSON.stringify(params.selector)})?.textContent||'(none)').trim()`, ); out.push(`[${params.selector}]: ${text.slice(0, 2000)}`); } else { const title = await evalJs(cdp, "document.title"); const url = await evalJs(cdp, "location.href"); out.push(`Title: ${title}`); out.push(`URL: ${url}`); } break; } case "wait": { const selector = params.selector; if (!selector) throw new Error("wait requires selector"); const timeout = params.timeout ?? 5000; const start = Date.now(); let found = false; while (Date.now() - start < timeout) { found = await evalJs(cdp, `!!document.querySelector(${JSON.stringify(selector)})`); if (found) break; await sleep(200); } out.push(found ? `Element appeared: ${selector}` : `Timed out waiting for ${selector}`); break; } case "screenshot": { const shot = await cdp.send("Page.captureScreenshot", { format: "png", captureBeyondViewport: false, }); const base64 = shot.data as string; if (signal?.aborted) return { content: [{ type: "text", text: "Cancelled" }], details: {} }; const file = join(tmpdir(), `pi-browser-${Date.now()}.png`); writeFileSync(file, Buffer.from(base64, "base64")); out.push(`Screenshot saved to: ${file}`); out.push(`Size: ${Math.round((base64.length * 0.75) / 1024)} KB`); break; } case "close": { cdp.close(); manager = null; out.push("Closed browser debugging session."); break; } default: throw new Error(`Unknown action: ${action}`); } return { content: [{ type: "text", text: out.join("\n") }], details: { action } }; }, }); }