import type { Protocol } from "devtools-protocol"; import { untilAborted } from "@oh-my-pi/pi-utils"; import { ToolError } from "@oh-my-pi/pi-tui/tools/tool-errors"; import type { ElementHandle, Frame, JSHandle, KeyboardTypeOptions, KeyInput, KeyPressOptions, MouseButton, Page, Realm, } from "puppeteer-core"; import { ToolAbortError, throwIfAborted } from "../tool-errors"; import { splitKeyCombo, ternKey } from "./tern/keys"; import { devtoolsFrameId } from "./frames"; import { untilAbortedOrReleased } from "./op-timeouts"; import { createFilePicker, openUploadSources, pickedFiles, refuseFolders, setFileInputPaths, type UploadFileInfo, type UploadRoute, type UploadSource, type UploadStaging, } from "./upload-files"; /** Options accepted by coordinate-based mouse clicks. */ export interface ClickAtOptions { /** Mouse button to press. */ button?: MouseButton; /** Number of clicks to dispatch. */ clickCount?: number; } /** Options accepted by pointer movement. */ export interface MouseMoveOptions { /** Number of intermediate movement events. */ steps?: number; } /** Options accepted by mouse button transitions. */ export interface MouseButtonOptions { /** Mouse button to press or release. */ button?: MouseButton; } /** Options accepted by element highlighting. */ export interface HighlightOptions { /** Time to keep the highlight visible in milliseconds. */ duration?: number; } /** Options accepted by element or page scrolling. */ export interface ScrollOptions { /** Scroll one matching element instead of the page. */ selector?: string; } /** A browser element handle with omp's additional interaction methods. */ export type InteractionHandle = ElementHandle & { dblclick(): Promise; check(): Promise; uncheck(): Promise; highlight(options?: HighlightOptions): Promise; }; interface ClickPoint { x: number; y: number; } type ActionabilityResult = { ok: true; x: number; y: number } | { ok: false; reason: string; coveredBy?: string }; interface PageRect { left: number; right: number; top: number; bottom: number; width: number; height: number; } interface PageRoot { host?: PageElement; } interface PageShadowRoot { elementFromPoint(x: number, y: number): PageElement | null; } /** An element on the way up to a contenteditable's editing host. */ interface EditableAncestor { isContentEditable: boolean; parentElement: EditableAncestor | null; focus(): void; contains(other: unknown): boolean; } interface PageElement { readonly tagName: string; id: string; readonly isConnected: boolean; type: string; checked: boolean; files: unknown; inert: boolean; readonly labels?: ArrayLike | null; matches(selector: string): boolean; readonly classList: ArrayLike; readonly dataset: Record; readonly style: Record; parentElement: PageElement | null; /** The `` rendering this node, its parent in the flat tree; `null` when unslotted. */ assignedSlot: PageElement | null; shadowRoot: PageShadowRoot | null; getBoundingClientRect(): PageRect; getClientRects(): ArrayLike; getRootNode(): PageRoot; contains(other: PageElement): boolean; getAttribute(name: string): string | null; setAttribute(name: string, value: string): void; dispatchEvent(event: unknown): boolean; scrollIntoView(options: { behavior: string; block: string; inline: string }): void; remove(): void; } interface PageDocument { elementFromPoint(x: number, y: number): PageElement | null; createElement(tag: string): PageElement; getElementById(id: string): PageElement | null; documentElement: { append(element: PageElement): void }; } interface PageGlobals { document: PageDocument; innerWidth: number; innerHeight: number; getComputedStyle(element: PageElement): { display: string; visibility: string; pointerEvents: string; opacity: string; }; Event: new (type: string, options: { bubbles: boolean; composed?: boolean }) => unknown; File: new (parts: Uint8Array[], name: string, options: { type: string; lastModified?: number }) => unknown; DataTransfer: new () => { items: { add(file: unknown): void }; files: unknown }; DragEvent: new (type: string, options: { bubbles: boolean; cancelable: boolean; dataTransfer: unknown }) => unknown; MessageChannel: new () => { port1: { onmessage: (() => void) | null; close(): void }; port2: { postMessage(message: null): void; close(): void }; }; } /** A click aborted while its element was still refused; `refusal` names the last failed check. */ export class ClickRefusedError extends ToolAbortError { constructor( readonly refusal: string, options?: ErrorOptions, ) { super(undefined, options); } } /** * Text entry aborted after input reached the page; `progress` says what the field may already hold. * The per-op timeout reports this instead of an actionability hint, since the element did take input. */ export class TypingInterruptedError extends ToolAbortError { constructor( readonly progress: string, options?: ErrorOptions, ) { super(undefined, options); } } function requireFiniteNumber(value: number, label: string): void { if (!Number.isFinite(value)) throw new ToolError(`${label} must be a finite number`); } /** * Page function (self-contained; serialisable via `.toString()`): whether `descendant` is `ancestor` or * renders inside it — its flat-tree ancestry, which follows slot assignment into shadow trees and climbs * shadow-root boundaries out to their hosts, where `Node.contains` stops. */ export function composedContains(ancestor: unknown, descendant: unknown): boolean { for (let current = descendant as PageElement | null; current;) { if (current === ancestor) return true; const root: PageRoot = current.getRootNode(); current = current.assignedSlot ?? current.parentElement ?? root.host ?? null; } return false; } interface ClickFlags { viaLabel: boolean; } /** Page half of {@link isClickActionable}; `contains` is {@link composedContains}, bound in by `clickActionableInPage`. */ function checkClickActionable( el: unknown, { viaLabel }: ClickFlags, contains: typeof composedContains, ): ActionabilityResult { const element = el as unknown as PageElement; const page = globalThis as unknown as PageGlobals; // A node the page replaced (e.g. a re-render) has no computed style; say so rather than misread it. if (!element.isConnected) { return { ok: false as const, reason: "detached (the page replaced this element; look it up again)" }; } const style = page.getComputedStyle(element); if (style.display === "none") return { ok: false as const, reason: "display:none" }; if (style.visibility === "hidden" || style.visibility === "collapse") { return { ok: false as const, reason: `visibility:${style.visibility}` }; } if (style.pointerEvents === "none") return { ok: false as const, reason: "pointer-events:none" }; // A custom checkbox or radio keeps its real control at opacity:0 under the box it draws; the hit test below decides. if ( Number(style.opacity) === 0 && !element.matches( 'input[type="checkbox"], input[type="radio"], [role="checkbox" i], [role="radio" i], [role="switch" i]', ) ) { return { ok: false as const, reason: "opacity:0" }; } const rect = element.getBoundingClientRect(); if (rect.width < 1 || rect.height < 1) return { ok: false as const, reason: "zero-size" }; // A wrapped link's box centre can fall between its lines, on the parent; aim at its first visible line. const fragments = Array.from(element.getClientRects()); const box = fragments.length === 0 ? rect : fragments.find( r => Math.min(page.innerWidth, r.right) - Math.max(0, r.left) >= 1 && Math.min(page.innerHeight, r.bottom) - Math.max(0, r.top) >= 1, ); if (!box) return { ok: false as const, reason: "off-viewport" }; const left = Math.max(0, Math.min(page.innerWidth, box.left)); const right = Math.max(0, Math.min(page.innerWidth, box.right)); const top = Math.max(0, Math.min(page.innerHeight, box.top)); const bottom = Math.max(0, Math.min(page.innerHeight, box.bottom)); if (right - left < 1 || bottom - top < 1) return { ok: false as const, reason: "off-viewport" }; const x = Math.floor((left + right) / 2); const y = Math.floor((top + bottom) / 2); let topElement = page.document.elementFromPoint(x, y); for (let depth = 0; topElement?.shadowRoot && depth < 16; depth++) { const nested = topElement.shadowRoot.elementFromPoint(x, y); if (!nested || nested === topElement) break; topElement = nested; } if (!topElement) return { ok: false as const, reason: "elementFromPoint-null" }; // A label forwards a click to its control unless the hit is other interactive content inside it. const interactiveContent = 'a[href], area[href], button, details, embed, iframe, input:not([type="hidden"]), label, select, textarea, summary, audio[controls], video[controls], [contenteditable=""], [contenteditable="true"]'; const hit: PageElement = topElement; const forwardedBy = (owner: PageElement): boolean => { let current: PageElement | null = hit; while (current && current !== owner) { if (current.matches(interactiveContent)) return false; current = current.assignedSlot ?? current.parentElement ?? current.getRootNode().host ?? null; } return current === owner; }; const onTarget = contains(element, topElement) || contains(topElement, element) || (viaLabel && Array.from(element.labels ?? []).some(forwardedBy)); if (!onTarget) { const tag = topElement.tagName.toLowerCase(); const id = topElement.id ? `#${topElement.id}` : ""; const classes = Array.from(topElement.classList) .slice(0, 2) .map(name => `.${name}`) .join(""); return { ok: false as const, reason: "covered", coveredBy: `<${tag}${id}${classes}>` }; } return { ok: true as const, x: x - rect.left, y: y - rect.top }; } /** {@link checkClickActionable} with {@link composedContains} bound in, as one self-contained page function. */ const clickActionableInPage = new Function( "el", "flags", `return (${checkClickActionable.toString()})(el, flags, ${composedContains.toString()});`, ) as (el: unknown, flags: ClickFlags) => ActionabilityResult; /** * Return a visible point relative to the element's box, or explain why it cannot receive a click. * A left click on one of the control's own `