/** * Spawn the Claude CLI as a subprocess driving the Playwright MCP server * for a single scenario. Returns the parsed verdict (if any) and the raw * subprocess output. * * Claude CLI must be installed on the user's PATH — a spawn ENOENT is mapped * to a friendly "not found" error rather than a raw stack trace. */ import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; import * as fs from 'node:fs'; import path from 'node:path'; import { sanitizeSteps, dropGeneratedRefAsserts, type TraceStep } from './scenario-cache'; export interface ScenarioVerdict { id: string; status: 'pass' | 'fail' | 'unknown'; reason: string; screenshot?: string; /** * Path to the recorded `.webm`. Present on the fast-replay path (Playwright * records it directly) AND on the AI record/heal path when `videoDir` is set — * @playwright/mcp records the agent's own browser session via the config's * `browser.contextOptions.recordVideo`, so a healed/recorded scenario still * yields a video in the SAME run, with no fragile re-replay. */ video?: string; } export interface DriveScenarioResult { verdict: ScenarioVerdict; /** Durable navigation skeleton the agent reported (empty if none emitted). */ trace: TraceStep[]; rawOutput: string; exitCode: number; durationMs: number; } export interface DriveScenarioOptions { prompt: string; outputDir: string; /** Pre-seeded session handed to the MCP browser via `--storage-state`. */ storageStatePath?: string; /** Hard cap per scenario; a wedged agent is killed and verdict'd unknown. */ timeoutMs?: number; /** * When set, the agent's browser records video into this dir via * @playwright/mcp's `recordVideo` config. The newest `.webm` emitted here is * returned on the verdict, giving the AI record/heal path a video on the same * run. Leave undefined for a screenshot-only run. */ videoDir?: string; /** Override for tests. Production callers leave undefined and get child_process.spawn. */ spawnImpl?: typeof spawn; /** Override for tests. Production leaves undefined and gets the real tree-killer. */ killImpl?: (child: Pick) => void; } /** * Kill the spawned `claude` process AND its descendants. `claude` spawns * `npx @playwright/mcp`, which spawns a headless Chromium; a SIGKILL on the * direct child alone leaves those grandchildren orphaned — each holding the live * `--storage-state` session the MCP browser loaded — so they outlive the run's * session-dir shred and, across scenarios, leak memory/handles on a CI box. * * `driveScenario` spawns with `detached: true`, making the child a process-group * leader; signalling the NEGATIVE pid reaps the whole group on POSIX. Windows has * no process groups, so we fall back to `taskkill /T` (tree). If the pid is * absent (test fakes) or the group is already gone, we fall back to `child.kill`. */ export const killProcessTree = ( child: Pick, deps: { platform?: NodeJS.Platform; processKill?: (pid: number, signal: NodeJS.Signals) => void; taskkill?: (pid: number) => void; } = {}, ): void => { const { platform = process.platform, processKill = (pid, signal) => process.kill(pid, signal), taskkill = (pid) => { // Absolute System32 path, never a bare PATH lookup, so a hijacked PATH can't // substitute a malicious `taskkill`. const system32 = path.join(process.env.SystemRoot ?? String.raw`C:\Windows`, 'System32', 'taskkill.exe'); spawnSync(system32, ['/pid', String(pid), '/T', '/F']); }, } = deps; const { pid } = child; if (typeof pid === 'number' && pid > 0) { try { if (platform === 'win32') { taskkill(pid); } else { processKill(-pid, 'SIGKILL'); } return; } catch { // Group already exited / not a leader → fall through to the direct kill. } } try { child.kill('SIGKILL'); } catch { // Already dead. } }; export const MCP_CONFIG_FILENAME = 'mcp.json'; export const PW_MCP_CONFIG_FILENAME = 'playwright-mcp-config.json'; const VERDICT_PREFIX = '::verdict::'; const TRACE_PREFIX = '::trace::'; const ALLOWED_TOOLS = 'mcp__playwright__*'; // Pin @playwright/mcp to an exact version. The whole record-and-replay contract // depends on the SHAPE of the a11y snapshot this server emits; `@latest` lets // that shape (and the recorded trace) drift silently between runs — a // reproducibility and supply-chain risk. Bump deliberately when validating a new // version against the funeral scenarios. const PW_MCP_VERSION = '0.0.76'; // A full multi-step issue flow (quote → policyholder → application → payment → // issue) needs noticeably longer to AI-record than a single-screen scenario: // observed ~280s wall to issue a main-member funeral policy, so 300s killed the // agent just after issuing but before it emitted its verdict. 600s gives the // longest flows headroom. Override per-run with `--scenario-timeout `. const DEFAULT_SCENARIO_TIMEOUT_MS = 600_000; export const writePlaywrightMcpConfig = (outputDir: string, storageStatePath?: string, videoDir?: string): string => { fs.mkdirSync(outputDir, { recursive: true }); const cfgPath = path.join(outputDir, MCP_CONFIG_FILENAME); // `--headless`: the record/heal browser runs invisibly, same as the // deterministic-replay path. The agent still captures screenshots + page // snapshots to --output-dir, so nothing is lost for debugging — but the user // no longer sees a browser window flash open/closed once per scenario. const args = ['-y', `@playwright/mcp@${PW_MCP_VERSION}`, '--headless', '--output-dir', outputDir]; if (storageStatePath) { // `--storage-state` is ONLY honoured for isolated (in-memory) sessions — // without `--isolated`, @playwright/mcp loads a persistent user-data-dir // profile and silently ignores the seeded session, so every scenario // starts logged out. The pair must always travel together. args.push('--isolated', '--storage-state', storageStatePath); } if (videoDir) { // There is no CLI flag for video; the only knob is the config file's // `browser.contextOptions.recordVideo`, which @playwright/mcp merges on top // of the CLI args. Every browser context the agent opens then records a // `.webm` into videoDir, flushed on context close (process exit). This is // what lets the AI record/heal path produce a video in the same run. fs.mkdirSync(videoDir, { recursive: true }); const pwCfgPath = path.join(outputDir, PW_MCP_CONFIG_FILENAME); fs.writeFileSync( pwCfgPath, JSON.stringify({ browser: { contextOptions: { recordVideo: { dir: videoDir } } } }, null, 2), ); args.push('--config', pwCfgPath); } const cfg = { mcpServers: { playwright: { command: 'npx', args, }, }, }; fs.writeFileSync(cfgPath, JSON.stringify(cfg, null, 2)); return cfgPath; }; /** One candidate payload parsed from a `::verdict::` marker, or null. */ const tryParseVerdictPayload = (payload: string, scenarioId: string): ScenarioVerdict | null => { try { const parsed = JSON.parse(payload) as Partial; return { id: parsed.id ?? scenarioId, status: parsed.status === 'pass' || parsed.status === 'fail' ? parsed.status : 'unknown', reason: parsed.reason ?? '', screenshot: parsed.screenshot, }; } catch { // Drift tolerance (mirrors sanitizeSteps / parseTrace salvage): the agent // sometimes emits a bare token like `::verdict:: PASS` instead of the JSON // envelope. Recognise an unambiguous pass/fail word before giving up, so a // genuinely-passing scenario isn't downgraded to `unknown` (which also means // it never caches). const token = payload.toLowerCase().replace(/[^a-z]/g, ''); if (token === 'pass' || token === 'passed') { return { id: scenarioId, status: 'pass', reason: 'salvaged bare verdict token' }; } if (token === 'fail' || token === 'failed') { return { id: scenarioId, status: 'fail', reason: 'salvaged bare verdict token' }; } return null; } }; /** * Pull the most recent parseable `::verdict::...` payload out of the stream. * Candidates are scanned LAST to FIRST and the first that parses wins — a long * run can re-emit the marker, and the final emission has been seen truncated * while an earlier one was complete. When a marker sits ALONE on its line * (observed live on main-life-spouse-child, 2026-07-03: the whole ~16-minute * record ended `unknown` with the empty reason "verdict JSON parse failed: "), * the JSON wrapped onto the following line — so a bare marker takes the next * non-empty line as its payload. */ export const parseVerdict = (rawOutput: string, scenarioId: string): ScenarioVerdict => { // Trim each line first: agents sometimes indent or markdown-wrap the marker. const lines = rawOutput.split(/\r?\n/).map((l) => l.trim()); const candidates: string[] = []; for (let i = 0; i < lines.length; i++) { if (!lines[i].startsWith(VERDICT_PREFIX)) continue; let payload = lines[i].slice(VERDICT_PREFIX.length).trim(); if (payload === '') { const next = lines.slice(i + 1).find((l) => l !== ''); payload = next ?? ''; } if (payload !== '') candidates.push(payload); } if (candidates.length === 0) { return { id: scenarioId, status: 'unknown', reason: 'inner agent did not emit a ::verdict:: line', }; } for (let i = candidates.length - 1; i >= 0; i--) { const verdict = tryParseVerdictPayload(candidates[i], scenarioId); if (verdict !== null) return verdict; } return { id: scenarioId, status: 'unknown', reason: `verdict JSON parse failed: ${(candidates.at(-1) ?? '').slice(0, 200)}`, }; }; /** Parse one candidate line into sanitized steps, or [] if it isn't a step array. */ const tryParseSteps = (payload: string): TraceStep[] => { let parsed: unknown; try { parsed = JSON.parse(payload); } catch { return []; } return dropGeneratedRefAsserts(sanitizeSteps(parsed)); }; /** * Coerce the agent's emitted trace into durable TraceSteps. A long flow can * re-emit `::trace::` more than once and end on a truncated/malformed line, so * don't trust only the literal LAST line: scan from last to first and take the * most recent one that parses to a non-empty step list. That salvages a good * mid-stream trace instead of discarding it (which would force a re-record). * * Salvage path: the agent sometimes emits the trace array WITHOUT the `::trace::` * prefix (prompt-adherence drift — it still emits `::verdict::`), which would * drop a perfectly good trace and force that scenario to re-record (heal) every * run. When no marked line yields steps, scan for a bare JSON array line whose * elements look like steps (`sanitizeSteps` keeps only well-formed `action` * objects, so a non-trace array collapses to []). This keeps a passing scenario * converging to an id-anchored cache instead of healing forever. */ /** A trace can only have issued a policy if it entered form data. */ const hasDataStep = (steps: TraceStep[]): boolean => steps.some((s) => s.action === 'fill' || s.action === 'select'); /** * From newest to oldest, return the first COMPLETE emission (one containing a * data-entry step — the signature of a real policy-issuing flow). A long run can * re-emit `::trace::` and end on a truncated stub (navigate/click-only) that * parses fine but would cache as a false 1-step success and replay forever * without issuing anything. Preferring a data-bearing emission steps over that * stub; only when NONE carries data do we fall back to the newest non-empty parse * (preserves the corrected-re-emit case: the agent's latest say wins). */ const pickRichestTrace = (candidates: string[]): TraceStep[] => { let newestNonEmpty: TraceStep[] = []; for (let i = candidates.length - 1; i >= 0; i--) { const steps = tryParseSteps(candidates[i]); if (steps.length === 0) continue; if (hasDataStep(steps)) return steps; if (newestNonEmpty.length === 0) newestNonEmpty = steps; } return newestNonEmpty; }; export const parseTrace = (rawOutput: string): TraceStep[] => { const allLines = rawOutput.split(/\r?\n/).map((l) => l.trim()); const marked = allLines.filter((l) => l.startsWith(TRACE_PREFIX)).map((l) => l.slice(TRACE_PREFIX.length).trim()); const fromMarked = pickRichestTrace(marked); if (fromMarked.length > 0) return fromMarked; // No usable marked line — salvage a bare (unprefixed) trace array. const bare = allLines.filter((l) => l.startsWith('[') && l.endsWith(']')); return pickRichestTrace(bare); }; /** * Pick the newest `.webm` in dir, or undefined if none. @playwright/mcp names * each recording with a random hash, so we can't predict the filename — but an * isolated single-context run emits exactly one, and newest-by-mtime is robust * even if a stale recording lingers. */ export const findNewestVideo = (videoDir: string | undefined): string | undefined => { if (!videoDir) return undefined; try { const webms = fs .readdirSync(videoDir) .filter((f) => f.endsWith('.webm')) .map((f) => { const full = path.join(videoDir, f); return { full, mtimeMs: fs.statSync(full).mtimeMs }; }) .sort((a, b) => b.mtimeMs - a.mtimeMs); return webms[0]?.full; } catch { // Dir never created (agent crashed before opening a context) → no video. return undefined; } }; export const driveScenario = async ( scenarioId: string, options: DriveScenarioOptions, ): Promise => { const { prompt, outputDir, storageStatePath, timeoutMs = DEFAULT_SCENARIO_TIMEOUT_MS, spawnImpl = spawn, killImpl = killProcessTree, videoDir, } = options; const mcpConfigPath = writePlaywrightMcpConfig(outputDir, storageStatePath, videoDir); const argv = [ '-p', prompt, '--allowedTools', ALLOWED_TOOLS, '--output-format', 'text', '--mcp-config', mcpConfigPath, ]; const startMs = Date.now(); return new Promise((resolve, reject) => { const child = spawnImpl('claude', argv, { cwd: outputDir, stdio: ['ignore', 'pipe', 'pipe'], // Become a process-group leader so a timeout can reap the whole // claude → npx → @playwright/mcp → chromium tree, not just `claude`. detached: true, }); let stdout = ''; let stderr = ''; // `settled` guards against double-resolution: on timeout we resolve // immediately (below) rather than waiting for `close`, because an orphaned // grandchild (npx → @playwright/mcp → chromium that escaped the process // group) can hold the inherited stdout pipe open for tens of minutes, so // `close` may never fire near the timeout. Resolving on the timer guarantees // the scenario is bounded by timeoutMs regardless of whether the kill fully // reaped the tree; the late `close` (if any) is then a no-op. let settled = false; const timer = setTimeout(() => { if (settled) return; settled = true; killImpl(child); resolve({ verdict: { id: scenarioId, status: 'unknown', reason: `scenario timed out after ${(timeoutMs / 1000).toFixed(0)}s and was killed`, }, trace: parseTrace(stdout), rawOutput: stdout + (stderr ? `\n[stderr]\n${stderr}` : ''), exitCode: 1, durationMs: Date.now() - startMs, }); }, timeoutMs); child.stdout?.on('data', (chunk: Buffer) => { stdout += chunk.toString(); }); child.stderr?.on('data', (chunk: Buffer) => { stderr += chunk.toString(); }); child.on('error', (err: Error & { code?: string }) => { clearTimeout(timer); if (settled) return; settled = true; if (err.code === 'ENOENT') { reject( new Error( 'Claude CLI (`claude`) not found on PATH. Install it from https://docs.claude.com/en/docs/claude-code and try again.', ), ); } else { reject(err); } }); child.on('close', (code) => { clearTimeout(timer); if (settled) return; settled = true; const durationMs = Date.now() - startMs; const rawOutput = stdout + (stderr ? `\n[stderr]\n${stderr}` : ''); const verdict = parseVerdict(stdout, scenarioId); // The agent recorded its own session via recordVideo; attach the .webm so // a healed/recorded scenario gets a video without a fragile re-replay. const video = findNewestVideo(videoDir); if (video) verdict.video = video; resolve({ verdict, trace: parseTrace(stdout), rawOutput, exitCode: code ?? 1, durationMs, }); }); }); };