import { execSync, spawn } from "node:child_process"; import { createServer } from "node:net"; import { setTimeout as delay } from "node:timers/promises"; import { chromium, type Browser, type BrowserContext, type CDPSession, type Page } from "playwright"; import { AgentError, type Observation, type WaitSpec, type WorkerInfo } from "../domain/types.ts"; import { shortId } from "../domain/ids.ts"; import { dataPaths, ensureDir } from "../store/paths.ts"; import { readWorkerInfo, writeWorkerInfo, clearWorkerInfo } from "../store/worker-info.ts"; import { BROWSER_LAUNCH_ID, chromeExecutable, isMissingChromeError, needsNoSandbox, persistentContextArgs, playwrightLaunchOverrides, resolveBrowserChannel, shouldReuseAttachedBrowser, type BrowserChannel, } from "./browser-channel.ts"; import { observePage, visibleText } from "./observe.ts"; import { openBackgroundPage } from "./background-page.ts"; import { clickWithoutOsFocus, fillWithoutOsFocus, scrollWithoutOsFocus, selectWithoutOsFocus, } from "../core/dom-gestures.ts"; const TAB_PREFIX = "bsa:"; export interface WorkerOptions { home: string; headless?: boolean; startUrl?: string; /** Installed Chrome or Playwright Chromium. Default is Chromium unless `BSA_BROWSER` is set. */ browser?: BrowserChannel; } /** Pointer/key events from the remote live view. x/y are 0–1 or CSS pixels. */ export type WorkerInputEvent = | { kind: "mouse"; action: "move" | "down" | "up" | "wheel"; x: number; y: number; button?: number; deltaY?: number; } | { kind: "key"; action: "down" | "up"; key: string; text?: string; modifiers?: number }; function childPids(): number[] { try { return execSync(`pgrep -P ${process.pid}`, { encoding: "utf8" }) .trim() .split(/\s+/) .map(Number) .filter(Boolean); } catch { return []; } } function chromePid(browser: Browser | null): number { if (!browser) return 0; const candidate = browser as Browser & { process?: (() => { pid?: number } | null) | { pid?: number }; }; if (typeof candidate.process === "function") { return candidate.process()?.pid ?? 0; } if (candidate.process && typeof candidate.process === "object") { return candidate.process.pid ?? 0; } return 0; } function isPidAlive(pid: number): boolean { try { process.kill(pid, 0); return true; } catch { return false; } } async function freePort(): Promise { return new Promise((resolve, reject) => { const server = createServer(); server.listen(0, "127.0.0.1", () => { const address = server.address(); const port = typeof address === "object" && address ? address.port : 0; server.close((err) => (err ? reject(err) : resolve(port))); }); server.on("error", reject); }); } async function waitForCdp(cdpUrl: string, attempts = 40): Promise { let last: unknown; for (let i = 0; i < attempts; i++) { try { const response = await fetch(`${cdpUrl}/json/version`); if (response.ok) return; last = new Error(`CDP ${response.status}`); } catch (err) { last = err; } await delay(100); } throw last instanceof Error ? last : new Error(`CDP not ready: ${cdpUrl}`); } async function connectCdp(cdpUrl: string, attempts = 15): Promise { let last: unknown; for (let i = 0; i < attempts; i++) { try { return await chromium.connectOverCDP(cdpUrl); } catch (err) { last = err; await delay(200); } } throw last instanceof Error ? last : new Error(`CDP connect failed: ${cdpUrl}`); } export class BrowserWorker { private readonly home: string; private readonly headless: boolean; private readonly browserChannel: BrowserChannel; private browser: Browser | null = null; private context: BrowserContext | null = null; private launchedHere = false; private trackedPids: number[] = []; private readonly pages = new Map(); private readonly consoleErrors = new Map(); private readonly lastObservation = new Map(); private info: WorkerInfo | null = null; private screencast: { client: CDPSession; tabId: string } | null = null; constructor(options: WorkerOptions) { this.home = options.home; this.headless = options.headless ?? false; this.browserChannel = resolveBrowserChannel({ explicit: options.browser, fallback: "chromium", }); } get headed(): boolean { return !this.headless; } get workerInfo(): WorkerInfo | null { return this.info; } async start(): Promise { if (this.context) return this.info!; const existing = await readWorkerInfo(this.home); if (existing?.cdpUrl) { try { await this.attachCdp(existing); if (shouldReuseAttachedBrowser(existing, { browser: this.browserChannel, launch: BROWSER_LAUNCH_ID })) { this.launchedHere = false; return this.info!; } await this.browser?.close().catch(() => undefined); } catch { // relaunch a fresh persistent context } this.browser = null; this.context = null; this.info = null; this.pages.clear(); } try { await this.launchManaged(); this.launchedHere = true; return this.info!; } catch (err) { const launched = this.context as BrowserContext | null; if (launched) { try { await launched.close(); } catch { // ignore } } this.context = null; this.browser = null; throw err; } } /** * Drop the Playwright CDP client without killing Chromium. * Leaves `worker.json` so a later BrowserWorker can connectOverCDP. * `stop()` still SIGKILLs via retained pid / trackedPids. * * Use `browser.close()` on a CDP-attached browser (not the private * `_connection.close()`): the latter leaves Chromium answering /json/version * but rejects later connectOverCDP. */ async disconnect(): Promise { await Promise.race([this.stopScreencast(), delay(300)]).catch(() => undefined); if (this.browser) { try { await this.browser.close(); } catch { // drop the CDP client only } } this.browser = null; this.context = null; this.pages.clear(); this.consoleErrors.clear(); this.lastObservation.clear(); this.launchedHere = false; // Keep this.info + trackedPids so stop() can still kill Chromium. } async stop(): Promise { await Promise.race([this.stopScreencast(), delay(300)]).catch(() => undefined); const fromDisk = this.info ?? (await readWorkerInfo(this.home).catch(() => null)); const pids = [...this.trackedPids, fromDisk?.pid ?? 0, this.info?.pid ?? 0, chromePid(this.browser)].filter( (pid) => pid > 0 && pid !== process.pid, ); // Quit Chromium over CDP while still attached so the profile flushes cookies. // Playwright `browser.close()` on a CDP session only drops the client. if (this.context) { try { const page = this.context.pages()[0] ?? (await this.context.newPage()); const session = await this.context.newCDPSession(page); await Promise.race([session.send("Browser.close" as never), delay(2_000)]); } catch { // fall through to signals } } else if (this.browser) { try { await Promise.race([this.browser.close(), delay(300)]); } catch { // already gone } } this.browser = null; this.context = null; this.pages.clear(); // Wait for a graceful CDP quit before escalating. const gracefulDeadline = Date.now() + 5_000; while (Date.now() < gracefulDeadline && pids.some((pid) => isPidAlive(pid))) { await delay(50); } for (const pid of new Set(pids)) { if (!isPidAlive(pid)) continue; try { process.kill(pid, "SIGTERM"); } catch { // already gone } } const termDeadline = Date.now() + 1_500; while (Date.now() < termDeadline && pids.some((pid) => isPidAlive(pid))) { await delay(50); } for (const pid of new Set(pids)) { try { process.kill(-pid, "SIGKILL"); } catch { try { process.kill(pid, "SIGKILL"); } catch { // already gone } } } this.launchedHere = false; this.trackedPids = []; this.info = null; await clearWorkerInfo(this.home); } /** * The persistent-profile context, and the pages already tracked in it. * * Exposed so `WorkerBrowserPort` can drive this browser with the same code that drives * a locally launched one. The alternative was a second implementation of click, type * and wait against the same Playwright API, which is exactly the duplication that let * a perception bug live in two places at once. * * This worker keeps ownership of process lifecycle, CDP reconnect, the profile * directory, screencast, and tab identity. Those are product concerns and none of them * belong to the agent. */ sharedContext(): BrowserContext { return this.requireContext(); } trackedPages(): Array<[string, Page]> { return [...this.pages.entries()]; } listTabs(): TabSnapshot[] { return [...this.pages.entries()].map(([tabId, page]) => ({ tabId, url: page.url(), title: page.url() === "about:blank" ? "" : "", })); } async describeTabs(): Promise> { const out = []; for (const [tabId, page] of this.pages) { out.push({ tabId, url: page.url(), title: await page.title() }); } return out; } async inspect(tabId?: string): Promise { const page = this.requirePage(tabId); const id = this.idOf(page); const observation = await observePage( page, id, this.lastObservation.get(id), this.consoleErrors.get(id) ?? [], ); this.lastObservation.set(id, observation); return observation; } async pageText(tabId?: string): Promise { return visibleText(this.requirePage(tabId)); } async screenshot(tabId: string | undefined, path: string): Promise { await this.requirePage(tabId).screenshot({ path, fullPage: false }); } async screenshotJpeg(tabId?: string): Promise<{ jpeg: string; tabId: string }> { const page = this.requirePage(tabId); const id = this.idOf(page); const buf = await page.screenshot({ type: "jpeg", quality: 50 }); return { jpeg: buf.toString("base64"), tabId: id }; } async navigate(tabId: string | undefined, url: string): Promise { await this.requirePage(tabId).goto(url, { waitUntil: "domcontentloaded" }); } async click(tabId: string | undefined, ref: string): Promise { const page = this.requirePage(tabId); const first = await this.inspect(this.idOf(page)); const named = first.controls.find((c) => c.ref === ref); if (!named) { throw new AgentError("missing_ref", `No control with ref ${ref}`, { ref }); } let lastErr: unknown; for (let attempt = 0; attempt < 4; attempt++) { try { if (named.role === "option" && named.name) { const option = page.getByRole("option", { name: named.name, exact: true }).first(); await option.waitFor({ state: "attached", timeout: 2_000 }); if (this.headed) await clickWithoutOsFocus(option, 5_000); else await option.click({ timeout: 5_000, force: true }); return; } if (attempt > 0) await this.inspect(this.idOf(page)); const locator = page.locator(`[data-bsa-ref="${cssEscape(ref)}"]`).first(); await locator.waitFor({ state: "attached", timeout: 2_000 }); if (this.headed) await clickWithoutOsFocus(locator, 5_000); else await locator.click({ timeout: 5_000, force: true }); return; } catch (err) { lastErr = err; const msg = err instanceof Error ? err.message : String(err); if (!/detached|not attached|not visible|Element is not/i.test(msg)) throw err; await delay(80); } } throw lastErr instanceof Error ? lastErr : new Error(String(lastErr)); } async type(tabId: string | undefined, ref: string, text: string): Promise { const page = this.requirePage(tabId); const observation = await this.inspect(this.idOf(page)); const control = observation.controls.find((c) => c.ref === ref); const locator = page.locator(`[data-bsa-ref="${cssEscape(ref)}"]`); if ((await locator.count()) === 0) { throw new AgentError("missing_ref", `No control with ref ${ref}`, { ref }); } const target = locator.first(); const useFill = control?.tag === "textarea" || control?.tag === "input"; if (useFill) { if (this.headed) await fillWithoutOsFocus(target, text, 5_000); else await target.fill(text); return control?.inputType; } await target.scrollIntoViewIfNeeded(); const box = await target.boundingBox(); if (box) { if (this.headed) await clickWithoutOsFocus(target, 3_000); else await page.mouse.click(box.x + box.width / 2, box.y + Math.min(box.height * 0.4, 120)); } else { if (this.headed) await clickWithoutOsFocus(target, 3_000); else await target.click({ force: true, timeout: 3_000 }); } let applied = false; for (let i = 0; i < 8 && !applied; i++) { applied = (await page.evaluate( `(() => { const api = window.monaco && window.monaco.editor && window.monaco.editor.getEditors && window.monaco.editor.getEditors()[0]; if (!api) return false; api.setValue(${JSON.stringify(text)}); return true; })()`, )) as boolean; if (!applied) await delay(200); } if (!applied) { await page.keyboard.press("Control+A"); await page.keyboard.insertText(text); } return control?.inputType; } async select(tabId: string | undefined, ref: string, value: string): Promise { const page = this.requirePage(tabId); await this.inspect(this.idOf(page)); const locator = page.locator(`[data-bsa-ref="${cssEscape(ref)}"]`); if ((await locator.count()) === 0) { throw new AgentError("missing_ref", `No control with ref ${ref}`, { ref }); } await (this.headed ? selectWithoutOsFocus(locator.first(), value, 5_000) : locator.first().selectOption(value)); } async scroll(tabId: string | undefined, ref?: string, dy = 600): Promise { const page = this.requirePage(tabId); if (ref) { await this.inspect(this.idOf(page)); const locator = page.locator(`[data-bsa-ref="${cssEscape(ref)}"]`); if ((await locator.count()) === 0) { throw new AgentError("missing_ref", `No control with ref ${ref}`, { ref }); } if (dy) { await scrollWithoutOsFocus(page, locator.first(), Number(dy), 5_000); return; } await locator.first().scrollIntoViewIfNeeded(); return; } if (this.headed) await scrollWithoutOsFocus(page, undefined, dy, 5_000); else await page.mouse.wheel(0, dy); } async wait(tabId: string | undefined, spec: WaitSpec): Promise { const page = this.requirePage(tabId); const timeout = Math.min(spec.timeoutMs ?? 5000, 15_000); switch (spec.kind) { case "load": await page.waitForLoadState("domcontentloaded", { timeout }); break; case "url": await page.waitForURL((url) => url.toString().includes(spec.value ?? ""), { timeout }); break; case "text": await page.getByText(spec.value ?? "", { exact: false }).first().waitFor({ timeout }); break; case "ref": await this.inspect(this.idOf(page)); await page.locator(`[data-bsa-ref="${cssEscape(spec.value ?? "")}"]`).first().waitFor({ timeout, }); break; case "timeout": await delay(timeout); break; } } async bringToFront(tabId: string): Promise { await this.requirePage(tabId).bringToFront(); } async openTab(url?: string): Promise { const context = this.requireContext(); const page = this.headless ? await context.newPage() : await openBackgroundPage(context); const tabId = await this.track(page); if (url) await page.goto(url, { waitUntil: "domcontentloaded" }); return tabId; } firstTabId(): string | undefined { return this.pages.keys().next().value; } controlInputType(tabId: string, ref: string): string | undefined { return this.lastObservation.get(tabId)?.controls.find((c) => c.ref === ref)?.inputType; } async startScreencast( onFrame: (jpeg: string, tabId: string) => void, tabId?: string, ): Promise { await this.stopScreencast(); const page = this.requirePage(tabId); const id = this.idOf(page); const client = await page.context().newCDPSession(page); await client.send("Page.startScreencast", { format: "jpeg", quality: 45, maxWidth: 1280, maxHeight: 720, everyNthFrame: 2, }); client.on("Page.screencastFrame", (event: { data: string; sessionId: number }) => { onFrame(event.data, id); void client.send("Page.screencastFrameAck", { sessionId: event.sessionId }).catch(() => undefined); }); this.screencast = { client, tabId: id }; return id; } async stopScreencast(): Promise { const current = this.screencast; this.screencast = null; if (!current) return; await Promise.race([ (async () => { try { await current.client.send("Page.stopScreencast"); } catch { // already stopped } try { await current.client.detach(); } catch { // already detached } })(), delay(300), ]); } async applyInput(event: WorkerInputEvent, tabId?: string): Promise { const page = this.requirePage(tabId ?? this.screencast?.tabId); const viewport = page.viewportSize() ?? { width: 1280, height: 720 }; if (event.kind === "mouse") { const x = event.x <= 1 ? event.x * viewport.width : event.x; const y = event.y <= 1 ? event.y * viewport.height : event.y; const button = event.button === 2 ? "right" : event.button === 1 ? "middle" : "left"; if (event.action === "move") { await page.mouse.move(x, y); return; } if (event.action === "down") { await page.mouse.move(x, y); await page.mouse.down({ button }); return; } if (event.action === "up") { await page.mouse.move(x, y); await page.mouse.up({ button }); return; } await page.mouse.move(x, y); await page.mouse.wheel(0, event.deltaY ?? 120); return; } if (event.action === "down") { if (event.text) { await page.keyboard.insertText(event.text); return; } await page.keyboard.down(event.key); return; } await page.keyboard.up(event.key); } /** * Spawn Chromium/Chrome as an OS process, then attach over CDP. * Playwright does not own the process, so `disconnect()` can drop the client * while leaving the profile browser alive for a later control process. */ private async launchManaged(): Promise { const paths = dataPaths(this.home); await ensureDir(paths.profileDir); const port = await freePort(); const overrides = playwrightLaunchOverrides(this.browserChannel); let executable: string; try { if (this.browserChannel === "chrome") { const chrome = await chromeExecutable(); if (!chrome) { throw new AgentError( "chrome_not_found", "Google Chrome is not installed. Install Chrome, or start with --chromium after `npx playwright install chromium`.", ); } executable = chrome; } else { executable = chromium.executablePath(); } } catch (err) { if (this.browserChannel === "chrome" && isMissingChromeError(err)) { throw new AgentError( "chrome_not_found", "Google Chrome is not installed. Install Chrome, or start with --chromium after `npx playwright install chromium`.", ); } throw err; } const args = [ `--user-data-dir=${paths.profileDir}`, ...(this.headless ? ["--headless=new"] : []), "--no-first-run", "--no-default-browser-check", "--window-size=1280,720", ...persistentContextArgs({ port, extraArgs: overrides.extraArgs, noSandbox: needsNoSandbox(), }), "about:blank", ]; const child = spawn(executable, args, { detached: true, stdio: "ignore", }); if (!child.pid) { throw new Error(`failed to spawn ${this.browserChannel}`); } child.unref(); const cdpUrl = `http://127.0.0.1:${port}`; try { await waitForCdp(cdpUrl); this.browser = await connectCdp(cdpUrl); this.context = this.browser.contexts()[0] ?? (await this.browser.newContext()); } catch (err) { try { process.kill(child.pid, "SIGKILL"); } catch { // already gone } throw err; } this.info = { pid: child.pid, cdpUrl, port, profileDir: paths.profileDir, startedAt: new Date().toISOString(), browser: this.browserChannel, launch: BROWSER_LAUNCH_ID, }; await writeWorkerInfo(this.home, this.info); this.trackedPids = [child.pid]; // Spawned by us; Playwright is only a CDP client (disconnect leaves Chromium). this.launchedHere = true; await this.hydratePages(); } private async attachCdp(info: WorkerInfo): Promise { this.browser = await connectCdp(info.cdpUrl); this.context = this.browser.contexts()[0] ?? (await this.browser.newContext()); this.info = info; await this.hydratePages(); } private async hydratePages(): Promise { const context = this.requireContext(); this.pages.clear(); for (const page of context.pages()) { await this.track(page); } if (this.pages.size === 0) { await this.track(await context.newPage()); } context.on("page", (page) => { void this.track(page); }); } private async track(page: Page): Promise { const prefix = JSON.stringify(TAB_PREFIX); const existing = (await page.evaluate( `window.name.startsWith(${prefix}) ? window.name.slice(${TAB_PREFIX.length}) : ""`, )) as string; const tabId = existing || shortId("tab"); const assigned = JSON.stringify(TAB_PREFIX + tabId); await page.evaluate( `if (!window.name.startsWith(${prefix})) window.name = ${assigned}`, ); this.pages.set(tabId, page); this.consoleErrors.set(tabId, []); page.on("console", (msg) => { if (msg.type() === "error") { const list = this.consoleErrors.get(tabId) ?? []; list.push(msg.text()); this.consoleErrors.set(tabId, list.slice(-20)); } }); page.on("pageerror", (err) => { const list = this.consoleErrors.get(tabId) ?? []; list.push(err.message); this.consoleErrors.set(tabId, list.slice(-20)); }); page.on("close", () => { this.pages.delete(tabId); }); return tabId; } private requireContext(): BrowserContext { if (!this.context) { throw new AgentError("worker_error", "Browser worker is not started"); } return this.context; } private requirePage(tabId?: string): Page { const id = tabId ?? this.firstTabId(); if (!id) { throw new AgentError("worker_error", "No browser tabs"); } const page = this.pages.get(id); if (!page) { throw new AgentError("unknown_tab", `Unknown tab ${id}`, { tabId: id }); } return page; } private idOf(page: Page): string { for (const [id, candidate] of this.pages) { if (candidate === page) return id; } throw new AgentError("unknown_tab", "Page is not tracked"); } } export interface TabSnapshot { tabId: string; url: string; title: string; } function cssEscape(value: string): string { return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"'); }