import type { Page } from 'playwright'; /** * Reads a page as a structured outline (cheap + reliable), the way * Playwright MCP / the accessibility tree works — NOT by sending pixels to a * vision model. Vision is a fallback for image-only pages (added later). * * Returns the interactive elements found, which the Screen Reader Agent then * enriches with MEANING and saves to the Zeta Core. */ export interface RawElement { role: string; label: string; selectorHint: string; expectedData?: string; } export async function readScreen(page: any): Promise { return page.evaluate(() => { // esbuild injects __name() for named arrow-function constants (e.g. `const priority = ...`). // Polyfill prevents ReferenceError when the serialized callback runs in the browser. if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn; const out: { role: string; label: string; selectorHint: string; expectedData?: string }[] = []; const seen = new Set(); // Noise patterns to skip — cookie banners, footer chrome, version strings // Intentionally excludes standalone "close" and "copy" — too many false positives // (Close account, Close preview, Copy link, Copy to clipboard are all valid test targets) const noiseLabel = /^(cookie|consent|privacy policy|terms of use|copyright|©|\bv\d+\.\d+|got it|dismiss|skip to|powered by|copied|\s*$)/i; // Include ARIA-role elements — Headless UI, Radix, Ant Design, Shadcn all use these instead of native tags const nodes = document.querySelectorAll( 'input:not([type=hidden]), select, textarea, button, a[href], ' + '[role="button"], [role="combobox"], [role="slider"], [role="switch"], ' + '[role="tab"], [role="menuitem"], [role="searchbox"], [role="spinbutton"]' ); nodes.forEach((el) => { const htmlEl = el as HTMLElement; // Skip invisible elements const style = window.getComputedStyle(htmlEl); if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return; const rect = el.getBoundingClientRect(); // Allow zero-size elements that carry an accessible label (sr-only icon buttons, skip-nav links) const hasAccessibleLabel = !!(el.getAttribute('aria-label') || el.getAttribute('aria-labelledby')); if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel) return; // Derive role const tag = el.tagName.toLowerCase(); const role = el.getAttribute('role') || (tag === 'input' ? ((el as HTMLInputElement).type || 'text') : tag); // Derive the best human-readable label (prefer aria > placeholder > inner text) const raw = ( el.getAttribute('aria-label') || el.getAttribute('placeholder') || (el.getAttribute('title') ?? '') || htmlEl.innerText?.replace(/\s+/g, ' ').trim() || el.getAttribute('name') || '' ).substring(0, 80); // Skip empty, noise patterns, or very long labels (likely container text) if (!raw || noiseLabel.test(raw) || raw.length > 70) return; // Deduplicate by role+normalised label const key = `${role}:${raw.toLowerCase()}`; if (seen.has(key)) return; seen.add(key); let expectedData: string | undefined; if (tag === 'input') { const input = el as HTMLInputElement; if (input.type === 'email' || /email/i.test(raw)) expectedData = 'valid_email'; else if (input.type === 'password' || /password/i.test(raw)) expectedData = 'password'; else if (input.type === 'tel' || /phone|mobile/i.test(raw)) expectedData = 'phone'; else if (input.type === 'date' || /\bdate\b/i.test(raw)) expectedData = 'date'; else if (/\b(code|otp|token)\b/i.test(raw)) expectedData = 'one_time_code'; else if (/\b(search|query)\b/i.test(raw)) expectedData = 'search_term'; } out.push({ role, label: raw, selectorHint: tag, expectedData }); }); // Prioritise: form fields > buttons > links; within each group keep order const priority = (e: typeof out[0]) => { if (['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role)) return 0; if (e.role === 'button' || e.role === 'submit') return 1; return 2; }; out.sort((a, b) => priority(a) - priority(b)); return out.slice(0, 25); // cap at 25 — enough context, not overwhelming }); }