/** * Element Locator * * Turns a CSS selector or a piece of visible text into page coordinates. * * Coordinates are what every input primitive ultimately needs — CDP dispatches * mouse and touch events at a point, not at an element. Resolving here means an * agent can say "click the Login button" without first taking a screenshot and * guessing pixels from it, and means the same query keeps working after the * page reflows. * * Every lookup sweeps all frames, not just the main one. `boundingBox()` is * already reported relative to the main frame, so an element inside an iframe * resolves to a directly clickable point with no offset arithmetic. */ import type { ElementHandle, Frame, Page } from 'puppeteer'; import type { BrowserActionTarget } from './types'; export interface LocatedElement { /** Centre of the element, in page coordinates — where input is aimed. */ x: number; y: number; box: { x: number; y: number; width: number; height: number }; tag: string; /** Trimmed visible text, capped for readability. */ text: string; /** A selector that addresses this element again, when one can be built. */ selector: string; role?: string; name?: string; href?: string; value?: string; placeholder?: string; type?: string; enabled: boolean; visible: boolean; /** True when the element lives inside an iframe. */ inFrame: boolean; frameUrl?: string; } export interface LocateQuery { selector?: string; text?: string; /** Filter by ARIA role or input type, e.g. "button", "link", "textbox". */ role?: string; nth?: number; visibleOnly?: boolean; limit?: number; /** Scroll the first match into view before measuring (default true). */ scrollIntoView?: boolean; } /** Metadata read out of the page for one element. */ interface ElementMeta { tag: string; text: string; selector: string; role?: string; name?: string; href?: string; value?: string; placeholder?: string; type?: string; enabled: boolean; visible: boolean; } /** * XPath that matches the *innermost* elements containing the text. * * Without the `not(.//*[...])` guard every ancestor up to matches too, * and the first hit would be the whole document rather than the button the * agent meant. */ function innermostTextXPath(text: string): string { const escaped = xpathLiteral(text.toLowerCase()); const lower = `translate(normalize-space(.), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')`; return `xpath///*[contains(${lower}, ${escaped}) and not(.//*[contains(${lower}, ${escaped})])]`; } /** XPath has no escape character; a mixed-quote string must be built with concat(). */ function xpathLiteral(value: string): string { if (!value.includes("'")) return `'${value}'`; if (!value.includes('"')) return `"${value}"`; const parts = value.split("'").map((part) => `'${part}'`); return `concat(${parts.join(`, "'", `)})`; } /** Read everything the caller might want to show, in one round trip per element. */ async function readMeta(handle: ElementHandle): Promise { return handle.evaluate((el) => { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); const visible = style.display !== 'none' && style.visibility !== 'hidden' && Number(style.opacity) !== 0 && rect.width > 0 && rect.height > 0; // Prefer a stable hook (id, test id, name) over a positional path so the // selector survives re-renders. const buildSelector = (node: Element): string => { if (node.id) return `#${CSS.escape(node.id)}`; const testId = node.getAttribute('data-testid') || node.getAttribute('data-test-id'); if (testId) return `[data-testid="${testId}"]`; const name = node.getAttribute('name'); if (name) return `${node.tagName.toLowerCase()}[name="${name}"]`; const path: string[] = []; let current: Element | null = node; while (current && current.nodeType === 1 && path.length < 5) { let part = current.tagName.toLowerCase(); const parent: Element | null = current.parentElement; if (parent) { const siblings = Array.from(parent.children).filter((c) => c.tagName === current!.tagName); if (siblings.length > 1) part += `:nth-of-type(${siblings.indexOf(current) + 1})`; } path.unshift(part); if (current.id) { path[0] = `#${CSS.escape(current.id)}`; break; } current = parent; } return path.join(' > '); }; const input = el as HTMLInputElement; const anchor = el as HTMLAnchorElement; return { tag: el.tagName.toLowerCase(), text: (el.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 120), selector: buildSelector(el), role: el.getAttribute('role') || undefined, name: el.getAttribute('aria-label') || el.getAttribute('name') || el.getAttribute('title') || undefined, href: anchor.href || undefined, value: typeof input.value === 'string' ? input.value.slice(0, 120) : undefined, placeholder: input.placeholder || undefined, type: input.type || undefined, enabled: !(el as HTMLButtonElement).disabled, visible }; }); } /** Collect candidate handles for a query, main frame first. */ async function collectHandles(page: Page, query: LocateQuery): Promise; frame: Frame }>> { const found: Array<{ handle: ElementHandle; frame: Frame }> = []; for (const frame of page.frames()) { if (frame.isDetached()) continue; try { const handles: ElementHandle[] = []; if (query.selector) { handles.push(...(await frame.$$(query.selector))); } else if (query.text) { handles.push(...(await frame.$$(innermostTextXPath(query.text)))); } else if (query.role) { // A role is expressed either explicitly or by the element itself: // `