import { access, mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; import path from "node:path"; import { assistantMessage, createBrowserFixture, getToolExecution, getToolExecutionDetails, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall, withBrowserFixture, } from "pi-coding-agent-test"; import { afterEach, describe, expect, test } from "vitest"; const extensionPath = path.join(path.dirname(import.meta.filename), "fixtures", "browser-decision-extension.ts"); const workspaces: string[] = []; const portCloseTimeoutMs = 2_000; const fileWaitTimeoutMs = 2_000; interface DecisionState { readonly port: number; readonly decision?: string; } afterEach(async () => { await Promise.all(workspaces.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); }); describe("dedicated browser fixture with real Pi", () => { test("submits a real page while PiIntegrationTest.run is pending and cleans every owner", async () => { const workspace = await makeWorkspace("pi-browser-decision-"); const fixture = await createBrowserFixture(); const environment = fixture.childEnvironment({ ...process.env, BROWSER: fixture.openerCommand }); const sentinelDirectory = firstPathEntry(environment); const run = startDecisionRun("browser-real-ui", workspace, "accepted-in-browser", environment); try { const url = await fixture.waitForUrl({ timeoutMs: 10_000 }); expect(await isPending(run)).toBe(true); expect(url).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/$/u); await fixture.page.goto(url); await fixture.page.getByLabel("decision-accepted-in-browser").fill("approve"); await fixture.page.getByRole("button", { name: "Submit" }).click(); const result = await run; expect(getToolExecution(result, "browser-decision-accepted-in-browser").isError).toBe(false); expect(getToolResultText(result, "browser-decision-accepted-in-browser")).toContain("approve"); expect(getToolExecutionDetails(getToolExecution(result, "browser-decision-accepted-in-browser"))) .toMatchObject({ decision: "approve" }); const state = await readDecisionState(workspace, "accepted-in-browser"); expect(state.decision).toBe("approve"); await expectPortClosed(state.port); expect(await sentinelFailures(sentinelDirectory)).toEqual([]); } finally { await fixture.close(); } expect(fixture.browser.isConnected()).toBe(false); await expect(access(sentinelDirectory)).rejects.toThrow(); }); test("keeps two pending Pi runs, URLs, pages, environments, and browser state isolated", async () => { const firstWorkspace = await makeWorkspace("pi-browser-first-"); const secondWorkspace = await makeWorkspace("pi-browser-second-"); const first = await createBrowserFixture(); const second = await createBrowserFixture(); const firstEnvironment = first.childEnvironment({ ...process.env, BROWSER: first.openerCommand, FIXTURE_SIDE: "first", }); const secondEnvironment = second.childEnvironment({ ...process.env, BROWSER: second.openerCommand, FIXTURE_SIDE: "second", }); const firstRun = startDecisionRun("browser-concurrent-first", firstWorkspace, "first", firstEnvironment); const secondRun = startDecisionRun("browser-concurrent-second", secondWorkspace, "second", secondEnvironment); try { const [firstUrl, secondUrl] = await Promise.all([ first.waitForUrl({ timeoutMs: 10_000 }), second.waitForUrl({ timeoutMs: 10_000 }), ]); expect(firstUrl).not.toBe(secondUrl); expect(first.context).not.toBe(second.context); expect(first.page).not.toBe(second.page); expect(firstPathEntry(firstEnvironment)).not.toBe(firstPathEntry(secondEnvironment)); await Promise.all([first.page.goto(firstUrl), second.page.goto(secondUrl)]); await first.page.evaluate(() => localStorage.setItem("fixture", "first")); await second.page.evaluate(() => localStorage.setItem("fixture", "second")); await first.page.getByLabel("decision-first").fill("first-choice"); await second.page.getByLabel("decision-second").fill("second-choice"); await Promise.all([ first.page.getByRole("button", { name: "Submit" }).click(), second.page.getByRole("button", { name: "Submit" }).click(), ]); const [firstResult, secondResult] = await Promise.all([firstRun, secondRun]); expect(getToolResultText(firstResult, "browser-decision-first")).toContain("first-choice"); expect(getToolResultText(secondResult, "browser-decision-second")).toContain("second-choice"); expect(await first.page.evaluate(() => localStorage.getItem("fixture"))).toBe("first"); expect(await second.page.evaluate(() => localStorage.getItem("fixture"))).toBe("second"); await expectPortClosed((await readDecisionState(firstWorkspace, "first")).port); await expectPortClosed((await readDecisionState(secondWorkspace, "second")).port); } finally { await Promise.all([first.close(), second.close()]); } }); test("routes a real application fallback through the failing sentinel and closes its session", async () => { const workspace = await makeWorkspace("pi-browser-fallback-"); const fixture = await createBrowserFixture(); const base = { ...process.env }; delete base.BROWSER; const environment = fixture.childEnvironment(base); delete environment.BROWSER; const sentinelDirectory = firstPathEntry(environment); try { const result = await startDecisionRun("browser-fallback-sentinel", workspace, "fallback", environment); const execution = getToolExecution(result, "browser-decision-fallback"); expect(execution.isError).toBe(true); expect(getToolResultText(result, "browser-decision-fallback")) .toMatch(/browser fallback failed|default|system|opener|fixture/u); expect(await sentinelFailures(sentinelDirectory)).not.toEqual([]); const state = await readDecisionState(workspace, "fallback"); await expectPortClosed(state.port); } finally { await fixture.close(); } await expect(access(sentinelDirectory)).rejects.toThrow(); }); test("honors explicit headed mode in the actual connected Chromium launch", async () => { if (process.env.PI_TEST_HEADED !== "1") { return; } await withBrowserFixture({ headless: false }, async (fixture) => { const session = await fixture.browser.newBrowserCDPSession(); try { const launch = await session.send("Browser.getBrowserCommandLine"); expect(launch.arguments.some((argument) => /^--headless(?:=|$)/u.test(argument))).toBe(false); expect(fixture.browser.isConnected()).toBe(true); await fixture.page.setContent(""); await fixture.page.getByRole("button").click(); await expect(fixture.page.getByRole("button").textContent()).resolves.toBe("headed browser"); } finally { await withTimeout(session.detach(), 1_000, "headed CDP session detach"); } }); }); test("closes browser resources and the decision server after a page assertion failure", async () => { const workspace = await makeWorkspace("pi-browser-assertion-"); let fixtureDirectory = ""; let browserConnected: (() => boolean) | undefined; await expect(withBrowserFixture({}, async (fixture) => { const environment = fixture.childEnvironment({ ...process.env, BROWSER: fixture.openerCommand }); fixtureDirectory = firstPathEntry(environment); browserConnected = () => fixture.browser.isConnected(); const run = startDecisionRun("browser-assertion-failure", workspace, "assertion", environment); const url = await fixture.waitForUrl({ timeoutMs: 10_000 }); await fixture.page.goto(url); await expect(fixture.page.getByLabel("decision-assertion").isVisible()).resolves.toBe(true); expect(await sentinelFailures(fixtureDirectory)).toEqual([]); await fixture.page.getByLabel("decision-assertion").fill("activity-complete"); await fixture.page.getByRole("button", { name: "Submit" }).click(); await run; throw new Error("intentional page assertion failure"); })).rejects.toThrow("intentional page assertion failure"); expect(browserConnected?.()).toBe(false); await expect(access(fixtureDirectory)).rejects.toThrow(); await expectPortClosed((await readDecisionState(workspace, "assertion")).port); }); test("attaches a real-boundary cleanup failure without replacing the page failure", async () => { const workspace = await makeWorkspace("pi-browser-cleanup-failure-"); let fixtureDirectory = ""; let browserConnected: (() => boolean) | undefined; let received: unknown; try { await withBrowserFixture({}, async (fixture) => { const environment = fixture.childEnvironment({ ...process.env, BROWSER: fixture.openerCommand }); fixtureDirectory = firstPathEntry(environment); browserConnected = () => fixture.browser.isConnected(); const run = startDecisionRun("browser-cleanup-failure", workspace, "cleanup-failure", environment); const url = await fixture.waitForUrl({ timeoutMs: 10_000 }); await fixture.page.goto(url); await fixture.page.getByLabel("decision-cleanup-failure").fill("complete"); await fixture.page.getByRole("button", { name: "Submit" }).click(); await run; const closeContext = fixture.context.close.bind(fixture.context); fixture.context.close = async () => { await closeContext(); throw new Error("real context cleanup detail"); }; throw new Error("primary real page assertion failure"); }); } catch (error) { received = error; } expect(describeError(received)).toContain("primary real page assertion failure"); expect(describeError(received)).toContain("real context cleanup detail"); expect(browserConnected?.()).toBe(false); await expect(access(fixtureDirectory)).rejects.toThrow(); await expectPortClosed((await readDecisionState(workspace, "cleanup-failure")).port); }); test("cleans the browser after a bounded Pi timeout", async () => { const workspace = await makeWorkspace("pi-browser-timeout-"); let fixtureDirectory = ""; let browserConnected: (() => boolean) | undefined; await expect(withBrowserFixture({}, async (fixture) => { const environment = fixture.childEnvironment({ ...process.env, BROWSER: fixture.openerCommand }); fixtureDirectory = firstPathEntry(environment); browserConnected = () => fixture.browser.isConnected(); const run = startDecisionRun("browser-pi-timeout", workspace, "timeout", environment, 10_000); const url = await fixture.waitForUrl({ timeoutMs: 10_000 }); await fixture.page.goto(url); await expect(fixture.page.getByLabel("decision-timeout").isVisible()).resolves.toBe(true); expect(await sentinelFailures(fixtureDirectory)).toEqual([]); expect(await isPending(run)).toBe(true); await run; })).rejects.toThrow(/Trace did not settle|timed out|timeout/u); expect(browserConnected?.()).toBe(false); await expect(access(fixtureDirectory)).rejects.toThrow(); await expectPortClosed((await readDecisionState(workspace, "timeout")).port); }); test("writes bounded diagnostics without secrets or unsafe browser artifacts", async () => { const artifactRoot = await makeWorkspace("pi-browser-diagnostics-"); const secret = "ALE44_SECRET_VALUE"; await withBrowserFixture({ artifactsDirectory: artifactRoot }, async (fixture) => { await fixture.page.setContent(``); await fixture.page.goto(`http://user:password@127.0.0.1:1/private?token=${secret}#fragment`).catch(() => undefined ); }); const files = await listFiles(artifactRoot); expect(files.length).toBeGreaterThan(0); expect(files.some((file) => /screenshot|trace/u.test(file))).toBe(false); const evidence = (await Promise.all(files.map((file) => readFile(file, "utf8")))).join("\n"); expect(Buffer.byteLength(evidence)).toBeLessThan(64 * 1024); expect(evidence).not.toContain(secret); expect(evidence).not.toContain("password"); expect(evidence).not.toContain("token="); expect(evidence).not.toContain("PLAYWRIGHT_BROWSERS_PATH"); expect(evidence).not.toMatch(/executablePath|userDataDir|authorization|cookie/iu); }); }); function startDecisionRun( testName: string, workspace: string, id: string, environment: NodeJS.ProcessEnv, timeoutMs = 20_000, ): ReturnType { return new PiIntegrationTest({ testName, artifactsDir: testArtifactsDir(import.meta.filename), cwd: workspace, extensions: [extensionPath], tools: ["browser_decision"], environment, timeoutMs, conversation: [ assistantMessage([ toolCall({ id: `browser-decision-${id}`, name: "browser_decision", arguments: { id } }), ], { stopReason: "toolUse" }), assistantMessage([text("The browser decision completed.")]), ], }).run("Open the local decision page and wait for the browser response"); } async function makeWorkspace(prefix: string): Promise { const workspace = await mkdtemp(path.join(tmpdir(), prefix)); workspaces.push(workspace); return workspace; } async function readDecisionState(workspace: string, id: string): Promise { const statePath = path.join(workspace, `browser-decision-${id}.json`); let state: DecisionState | undefined; await pollUntil( async () => { try { state = JSON.parse(await readFile(statePath, "utf8")) as DecisionState; return true; } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === "ENOENT") { return false; } throw error; } }, fileWaitTimeoutMs, `decision state ${statePath}`, ); if (state === undefined) { throw new Error(`Decision state was not readable after ${fileWaitTimeoutMs}ms: ${statePath}`); } return state; } async function isPending(promise: Promise): Promise { return await Promise.race([ promise.then(() => false, () => false), new Promise((resolve) => setTimeout(() => resolve(true), 50)), ]); } function firstPathEntry(environment: NodeJS.ProcessEnv): string { const entry = environment.PATH?.split(path.delimiter)[0]; if (entry === undefined || entry.length === 0) { throw new Error("Browser fixture did not prepend its sentinel directory to PATH"); } return entry; } async function sentinelFailures(directory: string): Promise { return (await readdir(directory)).filter((name) => /fail|marker|invoked/u.test(name)); } async function expectPortClosed(port: number): Promise { await pollUntil(async () => await canBind(port), portCloseTimeoutMs, `test-owned port ${port} to close`); } async function canBind(port: number): Promise { const server = createServer(); return await new Promise((resolve, reject) => { let cancelled = false; let listenSettled = false; let settlementStarted = false; const finish = (value: boolean | undefined, error?: unknown): void => { if (settlementStarted) { return; } settlementStarted = true; clearTimeout(timer); void closePortProbe(server, !listenSettled, port).then( () => { if (server.listening) { reject(new Error(`Port ${port} probe retained a listening handle after cleanup`)); } else if (error !== undefined) { reject(error); } else { resolve(value ?? false); } }, reject, ); }; const timer = setTimeout(() => { cancelled = true; finish(undefined, new Error(`single port ${port} closure check exceeded 250ms`)); }, 250); timer.unref(); server.once("listening", () => { listenSettled = true; if (cancelled) { void closePortProbe(server, false, port).catch(() => undefined); return; } finish(true); }); server.once("error", (error: NodeJS.ErrnoException) => { listenSettled = true; if (cancelled) { return; } if (error.code === "EADDRINUSE") { finish(false); } else { finish(undefined, error); } }); server.listen(port, "127.0.0.1"); }); } async function closePortProbe( server: ReturnType, waitForPendingClose: boolean, port: number, ): Promise { let closeCallbackError: Error | undefined; const closeEvent = new Promise((resolve) => server.once("close", resolve)); const closeCallback = new Promise((resolve) => { server.close((error) => { closeCallbackError = error; resolve(); }); }); if (waitForPendingClose) { await withTimeout(closeEvent, 250, `pending port ${port} probe close`); } else { await withTimeout(Promise.race([closeEvent, closeCallback]), 250, `port ${port} probe close`); } if ( closeCallbackError !== undefined && (closeCallbackError as NodeJS.ErrnoException).code !== "ERR_SERVER_NOT_RUNNING" ) { throw closeCallbackError; } if (server.listening) { await withTimeout( new Promise((resolve, reject) => server.close((error) => { if (error === undefined) { resolve(); } else { reject(error); } }) ), 250, `late port ${port} probe close`, ); } if (server.listening) { throw new Error(`Port ${port} probe retained a listening handle`); } } async function listFiles(directory: string): Promise { const entries = await readdir(directory, { withFileTypes: true }); const nested = await Promise.all(entries.map(async (entry) => { const target = path.join(directory, entry.name); return entry.isDirectory() ? await listFiles(target) : [target]; })); return nested.flat(); } async function pollUntil( predicate: () => boolean | Promise, timeoutMs: number, label: string, ): Promise { const deadline = Date.now() + timeoutMs; while (!await predicate()) { if (Date.now() >= deadline) { throw new Error(`Timed out after ${timeoutMs}ms waiting for ${label}`); } await new Promise((resolve) => setTimeout(resolve, 20)); } } async function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { let timer: NodeJS.Timeout | undefined; try { return await Promise.race([ promise, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error(`${label} exceeded ${timeoutMs}ms`)), timeoutMs); timer.unref(); }), ]); } finally { if (timer !== undefined) { clearTimeout(timer); } } } function describeError(error: unknown): string { if (error instanceof AggregateError) { return `${error.message} ${error.errors.map(describeError).join(" ")}`; } if (error instanceof Error) { return `${error.message} ${describeError(error.cause)}`; } return String(error ?? ""); }