/** * DOM Extractor — reads visible page content and correlates it with * the ARIA element cache for @e ref annotations. * * Contains: * - The inline EXTRACTOR_SCRIPT (runs in browser page context via evaluate) * - runExtractor() — calls the script and validates the result * - correlateElements() — matches extracted content against the element cache * - queryElementCache() — synchronous cache filtering * - correlateElements(), formatElementList(), formatRoleCountSummary() * - Boilerplate filtering constants */ import type { BrowserPlugin } from "../plugin-api.js"; import type { AriaCachedNode } from "./accessibility-tree.js"; import { roleIcon } from "./accessibility-tree.js"; // ─── Text length filter ──────────────────────────────────────────── /** * Minimum trimmed text length for an extracted element to be included. * * Set to 3 to capture short-but-meaningful labels like "FAQ", "Top", "Map" * while still filtering out true noise (single chars, whitespace-only text). * Visibility filters (offsetParent, getClientRects, visibility) handle most * garbage — lowering from 5 to 3 adds short categories/links that are clearly * intentional content. */ const MIN_TEXT_LENGTH = 3; // ─── Types ───────────────────────────────────────────────────────── /** Structured result from the DOM walker script. */ export interface ExtractResult { title: string; headings: Array<{ level: number; text: string }>; paragraphs: Array<{ text: string; region?: string }>; links: Array<{ text: string; href: string }>; images: Array<{ alt: string; src: string }>; /** Interactive elements (buttons, inputs, selects, textareas) */ interactive: Array<{ text: string; role: string; type?: string; disabled: boolean; }>; /** Error string if the extractor script caught an exception */ error?: string; } /** Output of the correlation step — text with @e annotations. */ interface CorrelatedResult { /** Formatted text output with @e annotations */ text: string; /** Number of matched refs */ matchedRefs: number; /** Whether a staleness notice was appended */ staleCache: boolean; } /** Outcome of runExtractor — discriminated union. */ type ExtractorOutcome = | { ok: true; result: ExtractResult } | { ok: false; error: string }; /** Parameters for queryElementCache. */ interface ElementCacheQuery { role?: string; name?: string; ref?: string; subtree?: string; } /** * Optional out-param for queryElementCache to signal when a ref was found * but an extra role/name/subtree filter rejected it. */ export interface QueryStatus { /** Set when filters.ref resolved to a cache node but an extra * role/name/subtree filter rejected it. */ refFilteredOut?: { node: AriaCachedNode; filter: "role" | "name" | "subtree"; value: string; }; } // ─── EXTRACTOR_SCRIPT — runs in browser page context ────────────── /** * Inline DOM walker script that runs in the browser page context * via page.evaluate(). Returns a JSON string matching ExtractResult. * * Playwright injects evaluate() at the DevTools protocol level, * bypassing page CSP. Sandboxed iframes without allow-scripts * are the only case that fails — the caller handles that. * * INVARIANT: EXTRACTOR_SCRIPT is pure DOM reads (no fetch/XHR/eval/ * writes). runExtractor() relies on this to pass read_only=true so * the bridge routes it through the stable isolated-world context on * Camoufox. If you add a write here, drop the `true` arg in * runExtractor(). */ export const EXTRACTOR_SCRIPT = `(() => { 'use strict'; try { const result = { title: document.title || '', headings: [], paragraphs: [], links: [], images: [], interactive: [], }; // ── Headings ── const headings = document.querySelectorAll('h1,h2,h3,h4,h5,h6'); for (const h of headings) { const text = h.innerText.trim(); if (text.length >= ${MIN_TEXT_LENGTH}) { result.headings.push({ level: +h.tagName[1], text: text }); } } // ── Paragraphs ── const paras = document.querySelectorAll('p, li, td, blockquote, figcaption'); for (const p of paras) { if (p.offsetParent === null) continue; if (p.getClientRects().length === 0) continue; if (window.getComputedStyle(p).visibility === 'hidden') continue; const text = p.innerText.trim(); if (text.length < ${MIN_TEXT_LENGTH}) continue; const region = p.closest('[role="region"], [aria-labelledby], [aria-label]'); const regionLabel = region ? (region.getAttribute('aria-label') || region.getAttribute('aria-labelledby') || '') : ''; result.paragraphs.push({ text: text, region: regionLabel || undefined }); } // ── Links ── const links = document.querySelectorAll('a[href]'); for (const a of links) { if (a.offsetParent === null) continue; if (a.getClientRects().length === 0) continue; if (window.getComputedStyle(a).visibility === 'hidden') continue; const text = a.innerText.trim(); if (text.length < ${MIN_TEXT_LENGTH}) continue; result.links.push({ text: text, href: a.href }); } // ── Images ── const imgs = document.querySelectorAll('img[alt]'); for (const img of imgs) { if (img.offsetParent === null) continue; if (img.getClientRects().length === 0) continue; if (window.getComputedStyle(img).visibility === 'hidden') continue; const alt = (img.alt || '').trim(); if (alt.length < ${MIN_TEXT_LENGTH}) continue; result.images.push({ alt: alt, src: img.src }); } // ── Interactive elements ── const interactive = document.querySelectorAll( 'button, [role="button"], input:not([type="hidden"]):not([type="radio"]):not([type="checkbox"]), select, textarea' ); for (const el of interactive) { if (el.offsetParent === null) continue; if (el.getClientRects().length === 0) continue; if (window.getComputedStyle(el).visibility === 'hidden') continue; const tagName = el.tagName.toLowerCase(); const role = el.getAttribute('role') || tagName; var text = ''; var disabled = false; if (el.disabled !== undefined) disabled = el.disabled; if (tagName === 'input') { const inputType = el.getAttribute('type') || 'text'; if (el.labels && el.labels.length > 0) { text = el.labels[0].innerText.trim(); } else if (el.placeholder) { text = el.placeholder; } result.interactive.push({ text: text, role: role, type: inputType, disabled: disabled }); } else if (tagName === 'select' || tagName === 'textarea') { if (el.labels && el.labels.length > 0) { text = el.labels[0].innerText.trim(); } else if (el.placeholder) { text = el.placeholder; } result.interactive.push({ text: text, role: role, disabled: disabled }); } else { text = (el.innerText || el.value || '').trim(); result.interactive.push({ text: text, role: role, disabled: disabled }); } } return JSON.stringify(result); } catch (err) { return JSON.stringify({ error: err.message || String(err) }); } })();`; // ─── runExtractor() — call the script via evaluate ──────────────── /** * Run the DOM extractor in the page context via plugin.evaluate(). * * The extractor runs purely on DOM properties — no fetch, no XHR, no eval. * Returns an ExtractorOutcome with the parsed result on success or an * error string on failure. */ export async function runExtractor( taskId: string, plugin: BrowserPlugin, ): Promise { try { const evalResult = await plugin.evaluate(taskId, EXTRACTOR_SCRIPT, true); if (!evalResult.success) { return { ok: false, error: evalResult.error ?? "evaluate failed (no error detail)", }; } // The script returns a JSON string inside result.result const rawJson = typeof evalResult.result === "string" ? evalResult.result : JSON.stringify(evalResult.result); let parsed: ExtractResult; try { parsed = JSON.parse(rawJson) as ExtractResult; } catch (e: unknown) { return { ok: false, error: `extractor returned invalid JSON: ${e instanceof Error ? e.message : String(e)}`, }; } // Check for script-level error if (parsed.error) { console.warn( "[pi-lean-portal] DOM extractor script error:", parsed.error, ); return { ok: false, error: parsed.error }; } // Validate shape (basic structural check) if ( typeof parsed.title !== "string" || !Array.isArray(parsed.headings) || !Array.isArray(parsed.paragraphs) || !Array.isArray(parsed.links) || !Array.isArray(parsed.images) || !Array.isArray(parsed.interactive) ) { return { ok: false, error: "extractor returned malformed result" }; } return { ok: true, result: parsed }; } catch { return { ok: false, error: "evaluate failed (no error detail)", }; } } // ─── correlateElements() — @e ref annotation ────────────────────── /** * Build a reverse index from the element cache: "role||name" → ref[]. * Handles many-to-many duplicates (multiple elements with same role+name). */ function buildReverseIndex( cache: Map, ): Map { const index = new Map(); for (const [, node] of cache) { const key = `${node.role}||${node.name}`; const existing = index.get(key); if (existing) { existing.push(node.ref); } else { index.set(key, [node.ref]); } } return index; } /** * Annotate extracted text with @e refs from the element cache. * * For each element in the extract result, looks up matching AriaCachedNode * by role + name. Annotates with @e refs where matches are found. * Handles duplicate role+name by annotating ALL matching @e refs. * * @param extracted - The structured result from the DOM walker * @param elementCache - The plugin's element cache (ref → AriaCachedNode) * @param cacheFresh - Whether the cache is still fresh (true = no staleness notice) */ export function correlateElements( extracted: ExtractResult, elementCache: Map, cacheFresh: boolean, ): CorrelatedResult { const reverseIndex = buildReverseIndex(elementCache); const matchedRefs = new Set(); const lines: string[] = []; // ── Title ── if (extracted.title) { lines.push(`Title: ${extracted.title}`); lines.push(""); } // ── Headings ── if (extracted.headings.length > 0) { lines.push("Headings:"); for (const h of extracted.headings) { const refs = reverseIndex.get(`heading||${h.text}`); const refStr = refs ? annotateRefs(refs, matchedRefs) : ""; lines.push(` ${refStr}📌 "${h.text}" [${h.level}]`); } lines.push(""); } // ── Content (paragraphs, links, images) ── if ( extracted.paragraphs.length > 0 || extracted.links.length > 0 || extracted.images.length > 0 ) { lines.push("Content:"); // Paragraphs for (const p of extracted.paragraphs) { const refs = reverseIndex.get(`paragraph||${p.text}`); const refStr = refs ? annotateRefs(refs, matchedRefs) : ""; const region = p.region ? ` [${p.region}]` : ""; // Cap individual paragraph text at 300 chars const displayText = p.text.length > 300 ? p.text.slice(0, 297) + "…" : p.text; lines.push(` ${refStr}${displayText}${region}`); } // Links for (const link of extracted.links) { const refs = reverseIndex.get(`link||${link.text}`); const refStr = refs ? annotateRefs(refs, matchedRefs) : ""; lines.push(` ${refStr}🔗 "${link.text}" → ${link.href}`); } // Images for (const img of extracted.images) { const refs = reverseIndex.get(`img||${img.alt}`); const refStr = refs ? annotateRefs(refs, matchedRefs) : ""; lines.push(` ${refStr}🖼 "${img.alt}"`); } lines.push(""); } // ── Interactive elements ── if (extracted.interactive.length > 0) { lines.push("Interactive:"); for (const el of extracted.interactive) { const refs = reverseIndex.get(`${el.role}||${el.text}`); const refStr = refs ? annotateRefs(refs, matchedRefs) : ""; const icon = roleIcon(el.role) || "• "; const disabledStr = el.disabled ? " [disabled]" : ""; const typeStr = el.type ? ` type="${el.type}"` : ""; const displayText = el.text && el.text.length > 0 ? ` "${el.text.length > 80 ? el.text.slice(0, 77) + "…" : el.text}"` : ""; lines.push( ` ${refStr}${icon}${el.role}${displayText}${typeStr}${disabledStr}`, ); } lines.push(""); } // ── Staleness notice ── let staleCache = false; if (!cacheFresh && matchedRefs.size > 0) { staleCache = true; lines.push( "⚠ Element refs may be stale — consider browser-snapshot before clicking.", ); } // ── Empty-output notice ── let text = lines.join("\n").trim(); if (text.length === 0) { text = "⚠ No extractable content found on the page. " + "The page may be empty, render behind a challenge, or block DOM queries. " + "Use browser-inspect { role/name/ref } against the element cache, " + "or browser-snapshot to inspect visually."; } return { text, matchedRefs: matchedRefs.size, staleCache, }; } /** * Annotate @e refs into a leading string. * Multiple refs are comma-separated. */ function annotateRefs(refs: string[], matchedRefs: Set): string { for (const ref of refs) { matchedRefs.add(ref); } if (refs.length === 1) { return `@${refs[0]} `; } return `@${refs.join(", @")} `; } // ─── queryElementCache() — synchronous cache filter ─────────────── /** * Query the element cache synchronously. * * - ref: look up a specific @e ref (with or without @ prefix) * - role: comma-separated roles, match any * - name: case-insensitive substring match * - subtree: find elements whose parentRef chain leads to a container * matching the given role * * All filters are AND-ed. */ export function queryElementCache( cache: Map, filters: ElementCacheQuery, status?: QueryStatus, ): AriaCachedNode[] { const results: AriaCachedNode[] = []; // Pre-compute subtree ancestry map once (shared by ref + loop paths) const subtreeAncestors = filters.subtree ? computeSubtreeAncestors(cache, filters.subtree) : null; // ref lookup: direct map access (cache keys are "e5" not "@e5") if (filters.ref) { const key = filters.ref.startsWith("@") ? filters.ref.slice(1) : filters.ref; const node = cache.get(key); if (node) { // Apply additional filters if any if (filters.role && !matchesRole(node, filters.role)) { if (status) status.refFilteredOut = { node, filter: "role", value: filters.role }; return []; } if (filters.name && !matchesName(node, filters.name)) { if (status) status.refFilteredOut = { node, filter: "name", value: filters.name }; return []; } if ( filters.subtree && subtreeAncestors && !subtreeAncestors.has(node.ref) ) { if (status) status.refFilteredOut = { node, filter: "subtree", value: filters.subtree, }; return []; } return [node]; } return []; } for (const [, node] of cache) { if (filters.role && !matchesRole(node, filters.role)) continue; if (filters.name && !matchesName(node, filters.name)) continue; if (subtreeAncestors && !subtreeAncestors.has(node.ref)) continue; results.push(node); } return results; } function matchesRole(node: AriaCachedNode, roleFilter: string): boolean { const roles = roleFilter.split(",").map((r) => r.trim().toLowerCase()); return roles.includes(node.role); } function matchesName(node: AriaCachedNode, nameFilter: string): boolean { return node.name.toLowerCase().includes(nameFilter.toLowerCase()); } /** * Compute the set of element refs that are inside a container matching * the given role. Follows parentRef chains. */ function computeSubtreeAncestors( cache: Map, containerRole: string, ): Set { const containerRefs = new Set(); const insideRefs = new Set(); // Find all container elements matching the role for (const [, node] of cache) { if (node.role === containerRole) { containerRefs.add(node.ref); } } if (containerRefs.size === 0) return insideRefs; // For each node, walk parentRef chain to see if any ancestor is a container for (const [, node] of cache) { // Skip the containers themselves if (containerRefs.has(node.ref)) continue; let current: AriaCachedNode | undefined = node; let depth = 0; while (current && depth < 100) { // Safety limit on chain depth if (containerRefs.has(current.ref)) { insideRefs.add(node.ref); break; } if (!current.parentRef) break; current = cache.get(current.parentRef); depth++; } } return insideRefs; } // ─── Output formatting ──────────────────────────────────────────── /** * Format a list of AriaCachedNode elements for agent consumption. */ export function formatElementList( elements: AriaCachedNode[], filters?: ElementCacheQuery, ): string { if (elements.length === 0) { if (filters?.ref) { return `Element @${filters.ref.replace(/^@/, "")} not found in cache. Use browser-snapshot to refresh.`; } return "No matching elements found."; } const lines: string[] = [ `Found ${elements.length} element${elements.length > 1 ? "s" : ""}:`, "", ]; for (const node of elements) { const icon = roleIcon(node.role); const refTag = `@${node.ref}`; const namePart = node.name ? ` "${node.name}"` : ""; const propsStr = node.props.length > 0 ? ` [${node.props.join(", ")}]` : ""; lines.push(` ${refTag} ${icon}${node.role}${namePart}${propsStr}`); } return lines.join("\n"); } /** * Format a role-count summary (fallback when no params provided). * * Only shows roles that have at least one element. Sorted by count descending. */ export function formatRoleCountSummary( cache: Map, ): string { const counts = new Map(); for (const [, node] of cache) { counts.set(node.role, (counts.get(node.role) ?? 0) + 1); } if (counts.size === 0) { return "No elements cached yet — use browser-snapshot to populate element cache."; } const sorted = Array.from(counts.entries()).sort((a, b) => b[1] - a[1]); const parts: string[] = []; for (const [role, count] of sorted) { const icon = roleIcon(role); parts.push(`${count} ${icon}${role}${count > 1 ? "s" : ""}`); } return ( parts.join(", ") + ". Use role=, name=, or text=true to query further." ); }