import { inspectTree, type InspectBridge } from './core' import { resolveTargetCoords, type ResolvedTargetCoords } from './resolve-target' import { waitForSootsimIdle } from './settling' import type { SimSemanticNode } from '../../../src/bridge-contract' import type { SootSimReplayTarget } from '@rnx/globals' export type TapFailure = 'not-found' | 'missed' | 'special' export type TapTargetSummary = SootSimReplayTarget export interface TapOutcome { payload: any result: any attempts: number failure?: TapFailure } export interface TapTextOptions { exact?: boolean role?: string | null within?: string | null minX?: number | null maxX?: number | null minY?: number | null maxY?: number | null near?: { x: number; y: number } | null nth?: number | null first?: boolean } export interface TapTimingOptions { initialWaitMs?: number deadlineMs?: number retryWaitMs?: number } type ScreenSize = { width: number; height: number } function isScreenSize(value: unknown): value is ScreenSize { if (value === null || typeof value !== 'object') return false const width = Reflect.get(value, 'width') const height = Reflect.get(value, 'height') return ( typeof width === 'number' && Number.isFinite(width) && width > 0 && typeof height === 'number' && Number.isFinite(height) && height > 0 ) } function viewportFromCapture(value: unknown): unknown { if (value === null || typeof value !== 'object') return undefined const tree = Reflect.get(value, 'tree') return tree !== null && typeof tree === 'object' ? Reflect.get(tree, 'viewport') : undefined } // the semantic tree has no covering root, so its first node is often a 64x64 // icon. capture.tree.viewport is the device layout size on local and cloud. // // a whole screen capture is the only way to ask for it today, and it is not // cheap: the headless runtime forces a full redraw of the tree before it walks // it, which measured 3.9-5.5s on a real app in a cloud box against 120-290ms // for every other command. that is why this runs on the failure path only. async function readScreen(bridge: InspectBridge): Promise { try { const viewport = viewportFromCapture( await bridge.send({ type: 'capture', includeVisual: false }), ) return isScreenSize(viewport) ? { width: viewport.width, height: viewport.height } : null } catch { return null } } function isOffscreen(cx: number, cy: number, screen: ScreenSize): boolean { return !(cx >= 0 && cy >= 0 && cx <= screen.width && cy <= screen.height) } const DEFAULT_AGENT_TIMING: Required = { initialWaitMs: 3000, deadlineMs: 5000, retryWaitMs: 700, } const DEFAULT_INTERACTIVE_TIMING: Required = { initialWaitMs: 1200, deadlineMs: 2500, retryWaitMs: 700, } function tapTiming(agent: boolean, opts: TapTimingOptions = {}) { return { ...(agent ? DEFAULT_AGENT_TIMING : DEFAULT_INTERACTIVE_TIMING), ...opts, } } export function isTapSuccess(result: any): boolean { if (!result || result.hit === false || result.ok === false) return false if (result.handled === false) return false if (result.requestedTargetMatched === false) return false if ( result.pointerTapHandled === false && !result.keyboardOpened && !result.isTextInput ) { return false } return true } export function tapTargetFromPayload( payload: any, textFallback?: string, ): TapTargetSummary { const id = payload.target?.id ?? payload.match?.id ?? payload.node?.id return { nodeId: payload.target?.nodeId ?? payload.match?.nodeId ?? payload.node?.nodeId ?? null, id: typeof id === 'string' ? id : null, testID: payload.target?.testID ?? payload.match?.testID ?? payload.node?.testID ?? null, text: payload.target?.text ?? payload.target?.accessibilityLabel ?? payload.match?.text ?? payload.match?.accessibilityLabel ?? payload.node?.text ?? textFallback ?? null, type: payload.target?.type ?? payload.match?.type ?? payload.node?.type ?? null, } } export async function tapResolvedTarget( bridge: InspectBridge, args: { textFallback?: string resolve: () => Promise agent?: boolean timing?: TapTimingOptions }, ): Promise { const timing = tapTiming(!!args.agent, args.timing) let attempts = 0 let lastPayload: any = null let lastResult: any = null // a CPU simulator only advances its animations while a command holds its // clock, so a sheet that was still sliding in when the last command returned // is still mid-slide now. settling first is what lets the resolved center and // the hit test below describe the same tree. const settleCloud = async () => { try { await bridge.send({ type: 'settle' }) } catch { // best-effort readiness gate; the retry loop decides success. } } if (bridge.plane === 'cloud') { await settleCloud() } else if (timing.initialWaitMs > 0) { try { await waitForSootsimIdle({ bridge, maxMs: timing.initialWaitMs, pollMs: 32, stablePolls: 2, }) } catch { // best-effort readiness gate; the retry loop decides success. } } const deadline = Date.now() + timing.deadlineMs let lastOffscreenAt: string | null = null // a cloud sim refuses a point outside its screen itself, and says so, so // asking it for a capture first would buy nothing. every other plane still // presses whatever the hit test finds at tree coordinates no finger can // reach, so there the screen has to be read before dispatching. const screen = bridge.plane === 'cloud' ? null : await readScreen(bridge) while (Date.now() <= deadline || attempts === 0) { attempts++ const resolved = await args.resolve() const payload = resolved && typeof resolved.cx === 'number' && typeof resolved.cy === 'number' && screen && isOffscreen(resolved.cx, resolved.cy, screen) ? { ...resolved, offscreen: true, screen } : resolved lastPayload = payload if ( payload?.error === 'bridge-not-ready' || payload?.ambiguous || payload?.nthOutOfRange ) { return { payload, result: null, attempts, failure: 'special' } } if (payload && typeof payload.cx === 'number' && typeof payload.cy === 'number') { // don't dispatch a point outside the device screen: the press would do // nothing a user could reproduce. report the offscreen miss instead so // drivers scroll first rather than getting stuck trusting a phantom hit. if (payload.offscreen) { lastResult = { hit: false, reason: 'offscreen', x: payload.cx, y: payload.cy, screen: payload.screen, } // stable offscreen position twice in a row is deterministic — the // node is scrolled out of view, not settling in. bail early instead // of retrying to the deadline. const at = `${Math.round(payload.cx)},${Math.round(payload.cy)}` if (lastOffscreenAt === at) { return { payload, result: lastResult, attempts, failure: 'missed' } } lastOffscreenAt = at } else { lastOffscreenAt = null const requestedTarget = tapTargetFromPayload(payload, args.textFallback) const result = await tapCoordinates( bridge, payload.cx, payload.cy, requestedTarget, ) // a cloud sim reports the offscreen refusal itself, carrying the point // and the screen in its diagnostics. give it the same shape the local // check produces so there is one offscreen failure to report. lastResult = result?.reason === 'offscreen' && result.diagnostics ? { ...result, hit: false, x: result.diagnostics.point?.x ?? payload.cx, y: result.diagnostics.point?.y ?? payload.cy, screen: result.diagnostics.screen, } : result if (result?.requestedTargetMatched === false) { return { payload, result, attempts, failure: 'missed' } } if (isTapSuccess(result)) return { payload, result, attempts } } } const remaining = deadline - Date.now() if (remaining <= 0) break if (bridge.plane === 'cloud') { if (lastResult?.reason !== 'target-moved' || attempts >= 3) break await settleCloud() continue } if (timing.retryWaitMs <= 0) break try { await waitForSootsimIdle({ bridge, maxMs: Math.min(remaining, timing.retryWaitMs), pollMs: 32, stablePolls: 2, }) } catch { await new Promise((resolve) => setTimeout(resolve, Math.min(120, remaining))) } } return { payload: lastPayload, result: lastResult, attempts, failure: lastPayload && typeof lastPayload.cx === 'number' ? 'missed' : 'not-found', } } export async function tapCoordinates( bridge: InspectBridge, x: number, y: number, target?: TapTargetSummary, ): Promise { return bridge.send({ type: 'tap', x, y, ...(target ? { target } : {}), }) } function payloadFromResolved( resolved: ResolvedTargetCoords, match: SootSimReplayTarget, extra: Record = {}, ) { return { cx: resolved.x, cy: resolved.y, match: { nodeId: match.nodeId ?? null, id: match.testID ?? null, testID: match.testID ?? null, text: match.text ?? null, type: match.type ?? null, }, target: { nodeId: resolved.nodeId ?? match.nodeId ?? null, id: resolved.testID ?? match.testID ?? null, testID: resolved.testID ?? match.testID ?? null, text: resolved.text ?? match.text ?? null, type: resolved.type ?? match.type ?? null, }, strategy: 'matched-node', ...extra, } } interface TextCandidate { node: SimSemanticNode ancestorTestIDs: string[] } // a non-pressable node repeating the text of its nearest labelled ancestor, // when that ancestor is pressable, is the ancestor's own caption (a button and // its text child), so only the button is a candidate. otherwise every labelled // button is an ambiguous pair. a pressable descendant is never folded into its // ancestor: same text does not prove it shares the ancestor's tap target. function collectTextCandidates( nodes: readonly SimSemanticNode[], ancestors: string[] = [], pressableOwnerText = '', ): TextCandidate[] { const out: TextCandidate[] = [] for (const node of nodes) { const text = nodeText(node) const caption = !!text && text === pressableOwnerText && !node.pressable if (!caption) out.push({ node, ancestorTestIDs: ancestors }) if (node.children?.length) { const next = node.testID ? [...ancestors, node.testID] : ancestors const owner = text ? (node.pressable ? text : '') : pressableOwnerText out.push(...collectTextCandidates(node.children, next, owner)) } } return out } function nodeText(node: SimSemanticNode): string { return node.text ?? node.label ?? '' } export async function tapById( bridge: InspectBridge, query: string, opts: { agent?: boolean; timing?: TapTimingOptions } = {}, ): Promise { return tapResolvedTarget(bridge, { agent: opts.agent, timing: opts.timing, resolve: async () => { const resolved = await resolveTargetCoords(bridge, { mode: 'testid', value: query }) if (!resolved) return { cx: null } return payloadFromResolved(resolved, resolved) }, }) } export async function tapByText( bridge: InspectBridge, query: string, options: TapTextOptions = {}, opts: { agent?: boolean; timing?: TapTimingOptions } = {}, ): Promise { return tapResolvedTarget(bridge, { agent: opts.agent, timing: opts.timing, textFallback: query, resolve: async () => { const { tree } = await inspectTree(bridge, 50) if (!Array.isArray(tree)) return { matched: 0, total: 0 } const needle = options.exact ? query : query.toLowerCase() let candidates = collectTextCandidates(tree).filter(({ node, ancestorTestIDs }) => { const text = nodeText(node) if (!text) return false if (options.exact ? text !== needle : !text.toLowerCase().includes(needle)) { return false } if (options.role && node.role !== options.role) return false if (options.within && !ancestorTestIDs.includes(options.within)) return false const ax = node.geometry.x const ay = node.geometry.y if (options.minX != null && !(ax >= options.minX)) return false if (options.maxX != null && !(ax <= options.maxX)) return false if (options.minY != null && !(ay >= options.minY)) return false if (options.maxY != null && !(ay <= options.maxY)) return false return true }) if (options.near) { const { x, y } = options.near candidates.sort( (a, b) => Math.hypot(a.node.geometry.x - x, a.node.geometry.y - y) - Math.hypot(b.node.geometry.x - x, b.node.geometry.y - y), ) } else { candidates.sort((a, b) => { if (Math.abs(a.node.geometry.y - b.node.geometry.y) > 2) { return a.node.geometry.y - b.node.geometry.y } return a.node.geometry.x - b.node.geometry.x }) } const total = candidates.length if (total === 0) return { matched: 0, total: 0 } let idx = 0 if (options.nth != null) { idx = options.nth < 0 ? total + options.nth : options.nth if (idx < 0 || idx >= total) { return { matched: 0, total, nthOutOfRange: true, nth: options.nth } } } else if (total > 1 && !options.first && !options.near) { return { ambiguous: true, total, candidates: candidates.slice(0, 10).map((candidate, index) => ({ idx: index, nodeId: candidate.node.nodeId, type: candidate.node.type, testID: candidate.node.testID ?? null, text: nodeText(candidate.node).slice(0, 60), abs: { x: candidate.node.geometry.x, y: candidate.node.geometry.y }, layout: { width: Math.round(candidate.node.geometry.width), height: Math.round(candidate.node.geometry.height), }, ancestorTestIDs: candidate.ancestorTestIDs.slice(0, 5), })), } } const picked = candidates[idx] if (!picked) return { matched: 0, total } const node = picked.node let resolved = node.testID ? await resolveTargetCoords(bridge, { mode: 'testid', value: node.testID }) : await resolveTargetCoords(bridge, { mode: 'text', value: nodeText(node) || query, }) if (!resolved) { resolved = { x: node.geometry.x + node.geometry.width / 2, y: node.geometry.y + node.geometry.height / 2, testID: node.testID ?? null, text: nodeText(node) || query, type: node.type, } } return payloadFromResolved(resolved, node, { total, idx }) }, }) } export async function tapBest( bridge: InspectBridge, query: string, opts: { agent?: boolean; timing?: TapTimingOptions } = {}, ): Promise { return tapResolvedTarget(bridge, { agent: opts.agent, timing: opts.timing, textFallback: query, resolve: async () => { const byId = await resolveTargetCoords(bridge, { mode: 'testid', value: query }) if (byId) { return { ...payloadFromResolved(byId, byId), strategy: 'testid', node: { nodeId: byId.nodeId ?? null, id: byId.testID ?? query, testID: byId.testID ?? query, type: byId.type ?? null, text: byId.text ?? null, }, } } const byText = await resolveTargetCoords(bridge, { mode: 'text', value: query }) if (byText) { return { ...payloadFromResolved(byText, byText), strategy: 'text', node: { nodeId: byText.nodeId ?? null, id: byText.testID ?? null, testID: byText.testID ?? null, type: byText.type ?? null, text: byText.text ?? query, }, } } return { strategy: 'none' } }, }) }