import { captureRenderedScreenshot, type RenderedScreenshotVisionPayload, } from './renderedScreenshots'; import axeSource from 'axe-core/axe.min.js?raw'; import { analyzeAccessibility, detectRenderedTargetMismatch, projectComputedStyles, projectContrast, projectDom, projectFocusOrder, projectLayout, projectResourceTiming, type AxeRunner, type PageProjectionParams, } from './pageProjections'; import type { V1BrowserCommandRunner, V1CommandExecutionCommand, V1CommandResult, V1RuntimeArtifact, } from './v1CommandRuntime'; type BrowserPageGetResult = Omit; const SCREENSHOT_PROJECTION = 'screenshot'; const DOCUMENT_PROJECTION_ROOTS = [ 'html', 'a11y', 'dom', 'computed_styles', 'layout', 'contrast', 'focus_order', 'resources', ] as const; type DocumentProjectionRoot = (typeof DOCUMENT_PROJECTION_ROOTS)[number]; type PageProjectionRoot = DocumentProjectionRoot | typeof SCREENSHOT_PROJECTION; interface PageProjectionRequest { root: PageProjectionRoot; params: PageProjectionParams; } interface BrowserPageGetRequest { targetUrl: string; viewport: { width: number; height: number }; waitMs: number; timeoutMs: number; projections: PageProjectionRequest[]; fullHeight: boolean; detail: 'auto' | 'low' | 'high'; } const DEFAULT_DESKTOP_VIEWPORT = { width: 1365, height: 900 }; const DEFAULT_MOBILE_VIEWPORT = { width: 390, height: 844 }; const DEFAULT_WAIT_MS = 800; const DEFAULT_TIMEOUT_MS = 12000; // Projections served from the faithfully rendered same-origin document. `screenshot` // is captured separately via captureRenderedScreenshot; all others read the rendered DOM. const DOCUMENT_PROJECTIONS: ReadonlySet = new Set(DOCUMENT_PROJECTION_ROOTS); const ALLOWED_PROJECTIONS = new Set([SCREENSHOT_PROJECTION, ...DOCUMENT_PROJECTIONS]); class BrowserCommandError extends Error { constructor(public code: string, message: string) { super(message); } } export function createBrowserCommandRunner(): V1BrowserCommandRunner { return { runBrowserCommand, }; } async function runBrowserCommand(command: V1CommandExecutionCommand): Promise { if (command.capability !== 'browser.page.get') { return { status: 'unsupported', artifacts: [], error: { code: 'unsupported_capability', message: `Unsupported browser capability: ${command.capability}`, }, }; } return runBrowserPageGet(command); } async function runBrowserPageGet(command: V1CommandExecutionCommand): Promise { try { const request = browserPageGetRequest(command.input); if (command.phase === 'validate') { return { status: 'succeeded', data: { valid: true }, artifacts: [], }; } const data: Record = { target: { url: request.targetUrl }, }; const artifacts: V1RuntimeArtifact[] = []; const documentProjections = request.projections.filter(projection => DOCUMENT_PROJECTIONS.has(projection.root), ); if (documentProjections.length > 0) { await withRenderedDocument(request, async (frameDocument, frameWindow) => { // Guard against silently analyzing the wrong page: if the iframe // rendered a 404/not-found page instead of the requested content // (e.g. a draft reached by its public permalink), report the mismatch // and skip the projections rather than describing the 404 page as if // it were the target. const mismatch = detectRenderedTargetMismatch(frameDocument, request.targetUrl); if (mismatch) { data.rendered_target_mismatch = mismatch; return; } const needsA11y = documentProjections.some(projection => projection.root === 'a11y'); const axeRunner = needsA11y ? axeRunnerForFrame(frameWindow, frameDocument) : undefined; for (const projection of documentProjections) { data[projection.root] = await computeDocumentProjection( projection, frameDocument, request.targetUrl, axeRunner, ); } }); } if (request.projections.some(projection => projection.root === SCREENSHOT_PROJECTION)) { const screenshot = await captureRenderedScreenshot({ url: request.targetUrl, viewport: request.viewport, wait_ms: request.waitMs, timeout_ms: request.timeoutMs, full_height: request.fullHeight, detail: request.detail, }); data.screenshot = screenshotSummary(screenshot); artifacts.push(browserScreenshotArtifact(command, request.targetUrl, screenshot)); } return { status: 'succeeded', data, artifacts, }; } catch (error) { return { status: 'failed', artifacts: [], error: { code: error instanceof BrowserCommandError ? error.code : 'browser_page_get_failed', message: error instanceof Error ? error.message : String(error), }, }; } } function browserPageGetRequest(inputValue: Record): BrowserPageGetRequest { const input = recordValue(inputValue); return { targetUrl: targetUrlForPageGet(input), viewport: viewportForPageGet(input), waitMs: positiveInteger(nestedNumber(input, ['wait', 'ms']) ?? input.wait_ms, DEFAULT_WAIT_MS), timeoutMs: positiveInteger( nestedNumber(input, ['wait', 'timeout_ms']) ?? input.timeout_ms, DEFAULT_TIMEOUT_MS, ), projections: selectedPageProjections(input.select), fullHeight: input.full_height !== false, detail: detailForPageGet(input), }; } async function computeDocumentProjection( projection: PageProjectionRequest, frameDocument: Document, targetUrl: string, axeRunner?: AxeRunner, ): Promise { switch (projection.root) { case 'html': return frameDocument.documentElement.outerHTML; case 'a11y': return { ...(await analyzeAccessibility(frameDocument, projection.params, axeRunner)), evaluated_url: targetUrl, }; case 'dom': return projectDom(frameDocument, projection.params); case 'computed_styles': return projectComputedStyles(frameDocument, projection.params); case 'layout': return projectLayout(frameDocument, projection.params); case 'contrast': return projectContrast(frameDocument, projection.params); case 'focus_order': return projectFocusOrder(frameDocument, projection.params); case 'resources': return projectResourceTiming(frameDocument.defaultView || window); default: return null; } } // Injects the bundled axe-core into the rendered frame and returns a runner that // invokes axe in the frame's own realm (cross-realm nodes fail axe's checks). function axeRunnerForFrame(frameWindow: Window, frameDocument: Document): AxeRunner | undefined { const frameAny = frameWindow as unknown as { axe?: { run: AxeRunner } }; if (!frameAny.axe) { const script = frameDocument.createElement('script'); script.textContent = axeSource; (frameDocument.head || frameDocument.documentElement).appendChild(script); script.remove(); } return frameAny.axe ? (context, options) => frameAny.axe!.run(context, options) : undefined; } // Renders the target once into a hidden same-origin iframe and hands the loaded // document to the caller; every document-backed projection reads this one render. async function withRenderedDocument( request: BrowserPageGetRequest, consume: (frameDocument: Document, frameWindow: Window) => Promise, ): Promise { const iframe = document.createElement('iframe'); iframe.setAttribute('aria-hidden', 'true'); iframe.tabIndex = -1; iframe.style.position = 'fixed'; iframe.style.left = '-10000px'; iframe.style.top = '0'; iframe.style.width = `${request.viewport.width}px`; iframe.style.height = `${request.viewport.height}px`; iframe.style.border = '0'; iframe.style.pointerEvents = 'none'; iframe.style.visibility = 'hidden'; document.body.appendChild(iframe); try { const loaded = waitForIframeLoad(iframe, request.timeoutMs); iframe.src = request.targetUrl; await loaded; const frameDocument = iframe.contentDocument; const frameWindow = iframe.contentWindow; if (!frameDocument?.documentElement || !frameWindow) { throw new Error('Rendered page target was not readable.'); } await frameDocument.fonts?.ready.catch(() => undefined); await sleep(request.waitMs); await consume(frameDocument, frameWindow); } finally { iframe.remove(); } } function targetUrlForPageGet(input: Record): string { const target = recordValue(input.target); const rawUrl = stringValue(target.url) || stringValue(target.permalink) || stringValue(target.post_url); if (rawUrl) return sameOriginUrl(rawUrl); const postId = positiveInteger(target.post_id ?? target.id, 0); if (postId > 0) return sameOriginUrl(`/?p=${postId}`); throw new BrowserCommandError('invalid_target', 'browser.page.get requires target.url or target.post_id.'); } function sameOriginUrl(value: string): string { const url = new URL(value, window.location.href); if (url.origin !== window.location.origin) { throw new BrowserCommandError('permission_denied', 'browser.page.get is limited to same-origin URLs.'); } return url.toString(); } function viewportForPageGet(input: Record): { width: number; height: number } { const viewport = recordValue(input.viewport); let defaultViewport = DEFAULT_DESKTOP_VIEWPORT; if (stringValue(viewport.name) === 'mobile') { defaultViewport = DEFAULT_MOBILE_VIEWPORT; } return { width: positiveInteger(viewport.width, defaultViewport.width), height: positiveInteger(viewport.height, defaultViewport.height), }; } function detailForPageGet(input: Record): 'auto' | 'low' | 'high' { const detail = stringValue(input.detail); return detail === 'low' || detail === 'high' ? detail : 'auto'; } function selectedPageProjections(rawSelect: unknown): PageProjectionRequest[] { const selected: PageProjectionRequest[] = []; const seen = new Set(); if (Array.isArray(rawSelect)) { for (const item of rawSelect) { const projectionRoot = projectionRootFromSelectPath(item); if (!isAllowedProjectionRoot(projectionRoot)) { throw new BrowserCommandError( 'invalid_command', 'browser.page.get select supports html, screenshot, a11y, dom, computed_styles, layout, contrast, focus_order, and resources projection paths.', ); } if (seen.has(projectionRoot)) continue; seen.add(projectionRoot); selected.push({ root: projectionRoot, params: projectionParamsFromSelectPath(item) }); } } else if (rawSelect !== undefined && rawSelect !== null) { throw new BrowserCommandError('invalid_command', 'browser.page.get select must be a list of projection paths.'); } if (selected.length < 1) selected.push({ root: 'html', params: {} }); return selected; } function projectionRootFromSelectPath(value: unknown): string | undefined { if (!Array.isArray(value) || value.length < 1) return undefined; const name = value[0]; return typeof name === 'string' ? name : undefined; } function isAllowedProjectionRoot(value: string | undefined): value is PageProjectionRoot { return Boolean(value && ALLOWED_PROJECTIONS.has(value)); } // Parameter-capable select paths follow the Query Document projection-path shape: // the first segment is the projection root and trailing key/value pairs scope it // (e.g. ['computed_styles', 'selector', '#promo'] or ['a11y', 'region', 'main']). function projectionParamsFromSelectPath(value: unknown): PageProjectionParams { const params: PageProjectionParams = {}; if (!Array.isArray(value)) return params; for (let index = 1; index + 1 < value.length; index += 2) { const key = value[index]; const param = value[index + 1]; if (typeof param !== 'string' || !param.trim()) continue; if (key === 'selector') params.selector = param.trim(); else if (key === 'region') params.region = param.trim(); } return params; } function screenshotSummary(screenshot: RenderedScreenshotVisionPayload): Record { return { status: 'captured', url: screenshot.url, width: screenshot.width, height: screenshot.height, mime: screenshot.mime, detail: screenshot.detail, }; } function browserScreenshotArtifact( command: V1CommandExecutionCommand, targetUrl: string, screenshot: RenderedScreenshotVisionPayload, ): V1RuntimeArtifact { return { id: `artifact_${command.command_id}_screenshot`, kind: 'rendered_screenshot', mime: screenshot.mime, size_bytes: approximateBase64DataUrlSizeBytes(screenshot.data_url), origin: { type: 'browser_command', command_id: command.command_id, capability: command.capability, url: targetUrl, }, visibility: 'llm_visible', retention: { policy: 'ephemeral_turn' }, llm_projection: { type: 'tool_image_bridge', detail: screenshot.detail, data_url: screenshot.data_url, }, }; } function waitForIframeLoad(iframe: HTMLIFrameElement, timeoutMs: number): Promise { return new Promise((resolve, reject) => { let done = false; const timer = window.setTimeout(() => { if (done) return; done = true; reject(new Error('Timed out while loading rendered page target.')); }, timeoutMs); iframe.onload = () => { if (done) return; done = true; window.clearTimeout(timer); resolve(); }; }); } function approximateBase64DataUrlSizeBytes(dataUrl: string): number { const encoded = dataUrl.split(',')[1] || ''; return Math.max(0, Math.floor((encoded.length * 3) / 4)); } function nestedNumber(source: Record, path: string[]): unknown { let current: unknown = source; for (const segment of path) { const currentRecord = recordValue(current); current = currentRecord[segment]; } return current; } function positiveInteger(value: unknown, fallback: number): number { const number = typeof value === 'number' ? value : Number(value); if (!Number.isFinite(number) || number <= 0) return fallback; return Math.round(number); } function stringValue(value: unknown): string | undefined { return typeof value === 'string' && value.trim() ? value : undefined; } function recordValue(value: unknown): Record { return value !== null && typeof value === 'object' && !Array.isArray(value) ? value as Record : {}; } function sleep(ms: number): Promise { return new Promise(resolve => window.setTimeout(resolve, ms)); }