// transport-agnostic rnx inspect-verb kernels. // // each function here sends the smallest bridge command its verb needs and // returns *structured data* — no arg parsing or stdout. the // CLI subcommand handlers wrap these with their WS bridge + text formatting; // the in-browser agent wraps them with an in-process bridge over its preview // window. one implementation, two transports — the DRY core of the inspect // surface. // // MUST stay browser-safe: no node builtins, no `process`, no `ws`. the only // dependency is the `InspectBridge.send` seam below, which both transports // satisfy (the CLI's `WsBridge` already has a matching `send`). import { HEADLESS_TENANT_FETCH_HUNG_MARKER, PUBLIC_NO_NETWORK_MARKER, type SemanticWaitResult, type WsBridgeCommand, } from '../../../src/bridge-contract' export { readyProbeHasContent, readyProbeHasTargetContent, } from '../../../src/bridge-contract' import type { SimSemanticNode, SimSemanticQueryResult, SimSemanticSelector, } from '../../../src/bridge-contract' import type { SootSimRouteInfo, SootSimRequestEntry, SootSimTimelineEvent, SootSimTimelineKind, SootSimTimelineQuery, SootSimTimelineQueryResult, SootSimTimelineSummary, } from '@rnx/globals' import type { KeyboardLayoutSnapshot } from 'sootsim-engine/render-worker/native-ui-protocol' // resolve a `--max-ms ` flag, tolerating the spellings agents and humans // actually type (`--maxMs`, `--maxms`, `--max_ms`). a silently-ignored // timeout flag is a sharp edge: the call doesn't error, it just uses the // default, so the operator thinks their bound applied when it didn't. export function resolveMaxMsFlag(args: string[], fallbackMs: number): number { const aliases = ['--max-ms', '--maxMs', '--maxms', '--max_ms'] for (const flag of aliases) { const i = args.indexOf(flag) if (i >= 0 && args[i + 1]) { const n = Number(args[i + 1]) if (Number.isFinite(n)) return Math.max(100, n) } } return fallbackMs } // the minimal bridge an inspect kernel needs: a JSON command channel into a // rnx instance. export interface InspectBridge { readonly plane?: 'local' | 'cloud' send(cmd: WsBridgeCommand, opts?: { timeoutMs?: number }): Promise } const INSPECT_PICK_COMMANDS = new Set([ 'tap', 'double-tap', 'tap-text', 'tap-id', 'long-press', 'touch', ]) export function isInspectPickCommand(subcommand: string | null | undefined): boolean { return typeof subcommand === 'string' && INSPECT_PICK_COMMANDS.has(subcommand) } export async function isInspectModeActive(bridge: InspectBridge): Promise { const active = await bridge.send({ type: 'evaluate', code: 'window.__sootsimEngineState?.inspectActive === true', }) return active === true } export async function shouldSkipAutoSettleForInspectPick( bridge: InspectBridge, subcommand: string | null | undefined, ): Promise { if (!isInspectPickCommand(subcommand)) return false try { return await isInspectModeActive(bridge) } catch { return false } } // `get count` — total SootSimNode count in the live tree. export async function inspectNodeCount( bridge: InspectBridge, ): Promise<{ nodes: number }> { const result: unknown = await bridge.send({ type: 'query', query: { kind: 'count' }, }) return { nodes: isSemanticQueryResult(result) ? result.total : 0 } } // `get tree` — pruned visible semantic tree. formatting stays client-side. export async function inspectTree( bridge: InspectBridge, depth = 5, ): Promise<{ depth: number; tree: unknown }> { const result: unknown = await bridge.send({ type: 'query', query: { kind: 'tree', depth }, }) const tree = isSemanticQueryResult(result) ? result.nodes : [] return { depth, tree } } // `get url` — the current page URL the bridge is attached to. export async function inspectUrl(bridge: InspectBridge): Promise<{ url: string }> { const url = await bridge.send({ type: 'evaluate', code: 'window.location.href' }) return { url: typeof url === 'string' ? url : '' } } // ─── describe ─── export interface DescribeDumpOptions { describe: boolean verbose: boolean filter: string testIdLike?: string onlyGlob?: string subtreeRoot?: string compact: boolean hideXy: boolean // machine consumers keep the whole string; the terminal dump clips it fullText?: boolean // depth for the recursive dump. omitted means the default 12. setting it // forces the recursive path, because the compact inspectable listing has no // depth to honor and would silently ignore the request. maxDepth?: number } export interface DescribeResult { tree?: string nodeCount?: number } function isSemanticGeometry(value: unknown): boolean { return ( value !== null && typeof value === 'object' && [ Reflect.get(value, 'x'), Reflect.get(value, 'y'), Reflect.get(value, 'width'), Reflect.get(value, 'height'), ].every((coordinate) => typeof coordinate === 'number' ? Number.isFinite(coordinate) : false, ) ) } export function isSimSemanticTree(value: unknown): value is SimSemanticNode[] { if (!Array.isArray(value)) return false return value.every((node) => { if ( node === null || typeof node !== 'object' || !Number.isInteger(Reflect.get(node, 'nodeId')) || typeof Reflect.get(node, 'type') !== 'string' || !isSemanticGeometry(Reflect.get(node, 'geometry')) ) { return false } for (const key of ['testID', 'text', 'role', 'label']) { const value = Reflect.get(node, key) if (value !== undefined && typeof value !== 'string') return false } const pressable = Reflect.get(node, 'pressable') if (pressable !== undefined && pressable !== true) return false const children = Reflect.get(node, 'children') return children === undefined || isSimSemanticTree(children) }) } function isSemanticQueryResult(value: unknown): value is SimSemanticQueryResult { const total = value !== null && typeof value === 'object' ? Reflect.get(value, 'total') : undefined return ( value !== null && typeof value === 'object' && typeof total === 'number' && Number.isInteger(total) && total >= 0 && isSimSemanticTree(Reflect.get(value, 'nodes')) ) } function globRegex(glob: string): RegExp { const escaped = glob.replace(/[.+^$(){}|[\]\\]/g, '\\$&') return new RegExp(`^${escaped.replace(/\*/g, '.*').replace(/\?/g, '.')}$`, 'i') } export function formatSemanticTree( nodes: readonly SimSemanticNode[], opts: Partial = {}, ): string { const filter = opts.filter?.toLowerCase() ?? '' const testIDRegex = opts.testIdLike ? globRegex(opts.testIdLike) : null const onlyRegex = opts.onlyGlob ? globRegex(opts.onlyGlob) : null const lines: string[] = [] const visit = (node: SimSemanticNode, indent: number): void => { const joined = [node.role, node.label, node.text, node.testID] .filter((value): value is string => typeof value === 'string') .join(' ') const matches = (!filter || joined.toLowerCase().includes(filter)) && (!testIDRegex || testIDRegex.test(node.testID ?? '')) && (!onlyRegex || onlyRegex.test(joined)) if (matches) { const parts = [ node.role ? `[${node.role}]` : node.pressable ? '[tap]' : `<${node.type}>`, ] if (node.text) { const text = opts.fullText || node.text.length <= 50 ? node.text : `${node.text.slice(0, 49).trimEnd()}…` parts.push(JSON.stringify(text)) } if (node.testID) parts.push(`#${node.testID}`) if (node.label && node.label !== node.text) { parts.push(`label=${JSON.stringify(node.label)}`) } if (!opts.hideXy) { const geometry = node.geometry parts.push( `@(${Math.round(geometry.x)},${Math.round(geometry.y)}) ${Math.round(geometry.width)}x${Math.round(geometry.height)}`, ) } if (node.pressable) parts.push('(tap)') lines.push(`${' '.repeat(indent + 1)}${parts.join(' ')}`) } for (const child of node.children ?? []) visit(child, indent + 1) } let roots = nodes if (opts.subtreeRoot) { const pending = [...nodes] let found: SimSemanticNode | undefined while (pending.length > 0 && !found) { const candidate = pending.shift() if (!candidate) continue if (candidate.testID === opts.subtreeRoot) found = candidate else pending.push(...(candidate.children ?? [])) } if (!found) return `__SUBTREE_NOT_FOUND__:${opts.subtreeRoot}` roots = [found] } for (const root of roots) visit(root, 0) return lines.join('\n') } // `describe` — format the visible structured tree for humans. callers own // arg-parsing, preflight, hints, and the watch loop. export async function inspectDescribe( bridge: InspectBridge, dumpOpts: DescribeDumpOptions, ): Promise { const dumpDepth = dumpOpts.maxDepth ?? 12 const result: unknown = await bridge.send({ type: 'query', query: { kind: 'tree', depth: dumpDepth }, }) if (!isSemanticQueryResult(result)) return {} return { tree: formatSemanticTree(result.nodes, dumpOpts), nodeCount: result.total, } } // ─── get-layout ─── // // per-element bounding-box measurement for meaningful elements on the current // screen. `describe` gives the element tree; this gives the numbers a designer // needs to check spacing/sizing parity: the on-screen box (x/y/w/h). callers opt // into computed borderRadius, per-side padding, fontSize, and visual style // fields with `styling:true` / `--styling`. // // the SAME LayoutElement shape is produced by both preview surfaces: // - native/rnx — `LAYOUT_GET_NATIVE_EVAL` reads the engine node tree via // __sootsimTest.listInspectable() (the source `describe` uses), where each // SootSimInspectInfo already carries absolute layout and optional style / // box-model data. // - web — the DOM extractor in webLayout.ts measures candidate elements with // getBoundingClientRect(), and calls getComputedStyle() only for styling. // both feed the same `formatLayoutElements` renderer, so the agent tool and the // CLI print identical output regardless of surface. export interface LayoutBox { x: number y: number w: number h: number } export interface LayoutPadding { t: number r: number b: number l: number } export interface LayoutExtractionOptions { styling?: boolean // case-insensitive substring matched against text, testID, role, and // selector. narrows a full-screen read down to the part being checked so // styling detail stays affordable. filter?: string } export interface LayoutTextContrast { ratio: number | null required: 3 | 4.5 foreground?: string background?: string reason?: string } export function scoreLayoutTextContrast(input: { normal: { width: number; height: number; data: ArrayLike } background: { width: number; height: number; data: ArrayLike } box: LayoutBox fontSize: number fontWeight?: number }): LayoutTextContrast { const required: 3 | 4.5 = input.fontSize >= 18 || (input.fontSize >= 14 && (input.fontWeight ?? 400) >= 700) ? 3 : 4.5 const { normal, background } = input if ( normal.width <= 0 || normal.height <= 0 || normal.width !== background.width || normal.height !== background.height ) { return { ratio: null, required, reason: 'capture dimensions do not match' } } const left = Math.max(0, Math.floor(input.box.x)) const top = Math.max(0, Math.floor(input.box.y)) const right = Math.min(normal.width, Math.ceil(input.box.x + input.box.w)) const bottom = Math.min(normal.height, Math.ceil(input.box.y + input.box.h)) if (right <= left || bottom <= top) { return { ratio: null, required, reason: 'text is outside the captured frame' } } const linear = (value: number) => { const channel = value / 255 return channel <= 0.04045 ? channel / 12.92 : Math.pow((channel + 0.055) / 1.055, 2.4) } const ratioOf = (foreground: readonly number[], behind: readonly number[]) => { const foregroundLuminance = 0.2126 * linear(foreground[0]) + 0.7152 * linear(foreground[1]) + 0.0722 * linear(foreground[2]) const backgroundLuminance = 0.2126 * linear(behind[0]) + 0.7152 * linear(behind[1]) + 0.0722 * linear(behind[2]) return ( (Math.max(foregroundLuminance, backgroundLuminance) + 0.05) / (Math.min(foregroundLuminance, backgroundLuminance) + 0.05) ) } const hex = (color: readonly number[]) => `#${color .slice(0, 3) .map((value) => Math.round(value).toString(16).padStart(2, '0')) .join('')}` const area = (right - left) * (bottom - top) const step = Math.max(1, Math.floor(Math.sqrt(area / 12_000))) const offsetAt = (x: number, y: number) => (y * normal.width + x) * 4 const deltaAt = (offset: number) => Math.max( Math.abs(normal.data[offset] - background.data[offset]), Math.abs(normal.data[offset + 1] - background.data[offset + 1]), Math.abs(normal.data[offset + 2] - background.data[offset + 2]), ) let strongestDelta = 0 let strongestForeground: number[] | null = null let opaqueBackground: number[] | null = null for (let y = top; y < bottom; y += step) { for (let x = left; x < right; x += step) { const offset = offsetAt(x, y) if (background.data[offset + 3] < 250) continue const behind = [ background.data[offset], background.data[offset + 1], background.data[offset + 2], ] opaqueBackground ??= behind const delta = deltaAt(offset) if (delta <= strongestDelta) continue strongestDelta = delta strongestForeground = [ normal.data[offset], normal.data[offset + 1], normal.data[offset + 2], ] } } let bestRatio = Infinity let bestForeground: number[] | null = null let bestBackground: number[] | null = null const consider = (foreground: number[], behind: number[]) => { const ratio = ratioOf(foreground, behind) if (ratio >= bestRatio) return bestRatio = ratio bestForeground = foreground bestBackground = behind } if (strongestDelta >= 2 && strongestForeground) { // The largest normal/background delta is the closest rendered sample to // the solid glyph color. Compare that color against every background pixel // where the suppressed pass proves text actually drew. This preserves the // weak side of text over gradients instead of selecting its best contrast. for (let y = top; y < bottom; y += step) { for (let x = left; x < right; x += step) { const offset = offsetAt(x, y) if (background.data[offset + 3] < 250 || deltaAt(offset) < 2) { continue } consider(strongestForeground, [ background.data[offset], background.data[offset + 1], background.data[offset + 2], ]) } } } else if (opaqueBackground) { // No final pixel changed when the renderer omitted this visible text. Its // effective foreground is therefore identical to the pixels behind it: // exact 1:1 contrast, not an "unmeasurable" escape hatch. consider(opaqueBackground, opaqueBackground) } if (!bestForeground || !bestBackground || !Number.isFinite(bestRatio)) { return { ratio: null, required, reason: 'could not isolate opaque rendered text pixels', } } return { ratio: Math.round(bestRatio * 100) / 100, required, foreground: hex(bestForeground), background: hex(bestBackground), } } export interface LayoutElement { // extractor-local identity and closest-first ancestry. app_get_layout uses // these internally to distinguish real overlaps from ordinary nesting; the // formatter intentionally omits them from the agent-facing rows. nodeId?: string | number ancestorIds?: Array // a stable way to refer to this element. testID/role when present; otherwise // a CSS selector (web) or the engine node type (native). selector?: string testID?: string role?: string type: string text?: string // on-screen bounding box in device/viewport px (x/y are absolute). box: LayoutBox // box-model fields are available only when the caller opts into styling. // measured zero remains distinct from an unavailable field. borderRadius?: number padding?: LayoutPadding fontSize?: number // styling — so a text-only agent (no vision) can still reason about visual // quality: theme correctness (is this screen actually emerald?), contrast // (color vs bg), washed-out opacity, garbled/struck text, clipping // (overflow:hidden + tiny box), flat-vs-styled (no bg where a card is // expected). colors are compacted css strings; each field is omitted when it // holds its boring default so rows stay scannable. color?: string bg?: string opacity?: number fontWeight?: number textDecoration?: string overflow?: string borderWidth?: number // only when the element actually draws a border. every element has a // resolved border-color, so reporting it unconditionally would be pure noise. borderColor?: string contrast?: LayoutTextContrast // text-flow metrics (web extractor) — wrap quality a no-vision agent cannot // otherwise see: how many lines the text broke into, whether the last line // is a lone hanging word, and whether ellipsis truncation is active. textLines?: number lastLineWords?: number // last line width relative to the widest line (0-1). lastLineWidthRatio?: number ellipsisActive?: boolean } export interface LayoutResult { preview: 'web' | 'native' count: number elements: LayoutElement[] } // in-page extractor for the native preview. mirrors the visibility + device // sizing the a11y / wait-ready kernels use so "what's on screen" agrees across // the inspect surface. returns LayoutElement[] (or [] when the bridge is cold). export function layoutGetNativeEval(opts: LayoutExtractionOptions = {}): string { const includeStyle = opts.styling === true const filter = opts.filter?.trim().toLowerCase() ?? '' return `(async () => { const t = window.__sootsimTest if (!t || typeof t.listInspectable !== 'function') return [] const list = await t.listInspectable({}) if (!Array.isArray(list)) return [] const includeStyle = ${JSON.stringify(includeStyle)} const scoreContrast = ${scoreLayoutTextContrast.toString()} // narrows by ELEMENT, not by style field: fewer rows, full detail on each. const filter = ${JSON.stringify(filter)} const matchesFilter = (n, text) => { if (!filter) return true const fields = [text, n.testID || n.testId, n.accessibilityRole || n.role, n.type] return fields.some((f) => typeof f === 'string' && f.toLowerCase().includes(filter)) } const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : 0) const round = (v) => Math.round(num(v)) // borderRadius can be authored as a string ("12px") or split per-corner; // prefer the uniform borderRadius, else the largest corner radius present. const radiusOf = (style) => { if (!style) return 0 const keys = ['borderRadius','borderTopLeftRadius','borderTopRightRadius','borderBottomLeftRadius','borderBottomRightRadius'] let max = 0 for (const k of keys) { const raw = style[k] const n = typeof raw === 'number' ? raw : (typeof raw === 'string' ? parseFloat(raw) : NaN) if (Number.isFinite(n)) max = Math.max(max, n) } return Math.round(max) } const fontSizeOf = (style) => { if (!style) return 0 const raw = style.fontSize const n = typeof raw === 'number' ? raw : (typeof raw === 'string' ? parseFloat(raw) : NaN) return Number.isFinite(n) ? Math.round(n) : 0 } // styling extractors — omit boring defaults so the agent only sees notable // styling (low opacity, struck text, clipping, a real border/color). const colorStr = (v) => { if (v == null) return undefined const s = String(v).split(' ').join('') return s && s !== 'transparent' && s !== 'rgba(0,0,0,0)' ? s : undefined } const opacityOf = (style) => { const o = style && style.opacity const n = typeof o === 'number' ? o : (typeof o === 'string' ? parseFloat(o) : NaN) return Number.isFinite(n) && n < 1 ? Math.round(n * 100) / 100 : undefined } const weightOf = (style) => { const w = style && style.fontWeight const n = w === 'bold' ? 700 : (typeof w === 'number' ? w : (typeof w === 'string' ? parseInt(w, 10) : NaN)) return Number.isFinite(n) && n >= 600 ? n : undefined } const decoOf = (style) => { const d = style && (style.textDecorationLine || style.textDecoration) return d && d !== 'none' && d !== 'normal' ? String(d).split(' ')[0] : undefined } const overflowOf = (style) => (style && style.overflow === 'hidden' ? 'hidden' : undefined) const borderWOf = (style) => { if (!style) return undefined const keys = ['borderWidth','borderTopWidth','borderBottomWidth','borderLeftWidth','borderRightWidth'] let max = 0 for (const k of keys) { const r = style[k] const n = typeof r === 'number' ? r : (typeof r === 'string' ? parseFloat(r) : NaN) if (Number.isFinite(n)) max = Math.max(max, n) } return max > 0 ? Math.round(max) : undefined } // only meaningful alongside a real border width, so it is resolved by the // caller below rather than on its own. const borderColorOf = (style) => { if (!style) return undefined const keys = ['borderColor','borderTopColor','borderBottomColor','borderLeftColor','borderRightColor'] for (const k of keys) { const c = colorStr(style[k]) if (c) return c } return undefined } const isVisible = (n) => { const layout = n && n.layout if (!layout || layout.width <= 0 || layout.height <= 0) return false const abs = (n && (n.absolute || n.absolutePosition)) || null if (!abs) return true const device = (n && n.device) || {} const screenW = Number(device.width) || window.innerWidth || 0 const screenH = Number(device.height) || window.innerHeight || 0 if (!screenW || !screenH) return true return abs.x + layout.width > 0 && abs.y + layout.height > 0 && abs.x < screenW && abs.y < screenH } const elements = list.filter(isVisible).filter((n) => { const t = typeof n.text === 'string' && n.text.trim() ? n.text.trim() : undefined return matchesFilter(n, t) }).map((n) => { const layout = n.layout || { x: 0, y: 0, width: 0, height: 0 } const abs = n.absolute || n.absolutePosition || { x: layout.x, y: layout.y } const text = typeof n.text === 'string' && n.text.trim() ? n.text.trim() : undefined const base = { ...(typeof n.nodeId === 'number' ? { nodeId: n.nodeId, ancestorIds: Array.isArray(n.ancestors) ? n.ancestors.map((ancestor) => ancestor && ancestor.nodeId).filter((id) => typeof id === 'number') : [], } : {}), testID: n.testID || n.testId || undefined, role: n.accessibilityRole || n.role || undefined, type: typeof n.type === 'string' ? n.type : 'node', text, box: { x: round(abs.x), y: round(abs.y), w: round(layout.width), h: round(layout.height) }, } if (!includeStyle) return base // computedStyle has the resolved values; style is the curated subset — // fall back across both so radius/fontSize survive either shape. const style = Object.assign({}, n.style || {}, n.computedStyle || {}) const pad = (n.boxModel && n.boxModel.padding) || {} const borderWidth = borderWOf(style) return { ...base, borderRadius: radiusOf(style), padding: { t: round(pad.top), r: round(pad.right), b: round(pad.bottom), l: round(pad.left) }, fontSize: text ? (fontSizeOf(style) || 14) : 0, color: text ? colorStr(style.color) : undefined, bg: colorStr(style.backgroundColor), opacity: opacityOf(style), fontWeight: weightOf(style), textDecoration: decoOf(style), overflow: overflowOf(style), borderWidth, borderColor: borderWidth ? borderColorOf(style) : undefined, } }) const textElements = elements.filter((element) => element.text && element.fontSize > 0) if (!includeStyle || textElements.length === 0) return elements const screenshot = window.SootSim && window.SootSim.bridges && window.SootSim.bridges.screenshot if (typeof screenshot !== 'function') { throw new Error('styled native layout requires the rnx capture bridge') } const device = list.find((node) => node && node.device)?.device || {} const width = Math.round(Number(device.width) || window.innerWidth || 0) const height = Math.round(Number(device.height) || window.innerHeight || 0) if (!width || !height) throw new Error('native layout has no capture dimensions') const capture = async (suppressText) => { const dataUrl = await screenshot({ format: 'png', outputWidth: width, outputHeight: height, ...(suppressText ? { suppressText: true } : {}), }) const image = await new Promise((resolve, reject) => { const next = new Image() next.onload = () => resolve(next) next.onerror = () => reject(new Error('could not decode native layout pixels')) next.src = dataUrl }) const canvas = document.createElement('canvas') canvas.width = image.naturalWidth canvas.height = image.naturalHeight const context = canvas.getContext('2d', { willReadFrequently: true }) if (!context) throw new Error('could not read native layout pixels') context.drawImage(image, 0, 0) return context.getImageData(0, 0, canvas.width, canvas.height) } const normal = await capture(false) const background = await capture(true) for (const element of textElements) { element.contrast = scoreContrast({ normal, background, box: element.box, fontSize: element.fontSize, fontWeight: element.fontWeight, }) } return elements })()` } export const LAYOUT_GET_NATIVE_EVAL = layoutGetNativeEval() // `get-layout` (native) — bounding boxes for visible meaningful elements on the // rnx/native preview. transport-agnostic: the CLI runs it over its WS // bridge, the agent runs it over its in-process preview bridge. export async function inspectGetLayout( bridge: InspectBridge, opts: LayoutExtractionOptions = {}, ): Promise { if (!opts.styling) { const result: unknown = await bridge.send({ type: 'query', query: { kind: 'find', selector: { visible: true } }, }) if (!isSemanticQueryResult(result)) return [] const filter = opts.filter?.trim().toLowerCase() ?? '' return result.nodes .filter((node) => { if (!filter) return true return [node.text, node.testID, node.role, node.type].some( (value) => typeof value === 'string' && value.toLowerCase().includes(filter), ) }) .map((node) => ({ nodeId: node.nodeId, ...(node.testID ? { testID: node.testID } : {}), ...(node.role ? { role: node.role } : {}), type: node.type, ...(node.text ? { text: node.text } : {}), box: { x: Math.round(node.geometry.x), y: Math.round(node.geometry.y), w: Math.round(node.geometry.width), h: Math.round(node.geometry.height), }, })) } const raw = await bridge.send({ type: 'evaluate', code: layoutGetNativeEval(opts), }) return Array.isArray(raw) ? (raw as LayoutElement[]) : [] } // one-line-per-element renderer shared by the CLI and the agent tool so web + // native + every transport print the same thing. padding collapses to a single // number when uniform, and zero-valued fields are omitted to keep rows scannable. export function formatLayoutElements( elements: readonly LayoutElement[], opts?: { styling?: boolean }, ): string { if (elements.length === 0) return ' no visible elements found' const lines = elements.map((el) => { const ident = el.testID ? `#${el.testID}` : el.selector ? el.selector : el.role ? `[${el.role}]` : `<${el.type}>` const parts = [ident, `@(${el.box.x},${el.box.y})`, `${el.box.w}x${el.box.h}`] if (el.borderRadius !== undefined && el.borderRadius > 0) { parts.push(`radius:${el.borderRadius}`) } const p = el.padding if (p && (p.t || p.r || p.b || p.l)) { const uniform = p.t === p.r && p.r === p.b && p.b === p.l parts.push(uniform ? `pad:${p.t}` : `pad:${p.t},${p.r},${p.b},${p.l}`) } if (el.fontSize !== undefined && el.fontSize > 0) { parts.push(`font:${el.fontSize}`) } // styling is opt-in (the `styling` flag): the default rows stay scannable // for UI inspection, and a no-vision agent asks for styling when judging // visual quality (contrast, washed-out, garbled/struck, clipping, flat). if (opts?.styling) { if (el.fontWeight) parts.push(`weight:${el.fontWeight}`) if (el.color) parts.push(`color:${el.color}`) if (el.bg) parts.push(`bg:${el.bg}`) if (el.contrast) { const score = el.contrast.ratio == null ? `?/${el.contrast.required}` : `${el.contrast.ratio}/${el.contrast.required}${el.contrast.ratio < el.contrast.required ? ' FAIL' : ''}` parts.push(`contrast:${score}`) if (el.contrast.background) parts.push(`on:${el.contrast.background}`) } if (el.opacity != null) parts.push(`opacity:${el.opacity}`) if (el.textDecoration) parts.push(`deco:${el.textDecoration}`) if (el.overflow) parts.push(`overflow:${el.overflow}`) if (el.borderWidth) { parts.push( el.borderColor ? `border:${el.borderWidth}/${el.borderColor}` : `border:${el.borderWidth}`, ) } } if (el.text) { const text = el.text.length > 40 ? `${el.text.slice(0, 39)}…` : el.text parts.push(`"${text}"`) } return ` ${parts.join(' ')}` }) return lines.join('\n') } // ─── accessibility ─── export interface AccessibilityTreeNode { role: string label: string | null hint: string | null state: Record | null testID: string | null position: { x: number; y: number } | null size: { w: number; h: number } | null } // `get a11y` — flat accessibility summary projected from the visible semantic // tree, which already excludes hidden and covered content. export async function inspectAccessibilityTree( bridge: InspectBridge, ): Promise { const result: unknown = await bridge.send({ type: 'query', query: { kind: 'find', selector: { visible: true } }, }) if (!isSemanticQueryResult(result)) return [] return result.nodes.flatMap((node) => { if (!node.role && !node.label && !node.text && !node.pressable) return [] return [ { role: node.role ?? (node.pressable ? 'button' : node.type === 'text' ? 'statictext' : 'none'), label: node.label ?? node.text ?? null, hint: null, state: null, testID: node.testID ?? null, position: { x: Math.round(node.geometry.x), y: Math.round(node.geometry.y), }, size: { w: Math.round(node.geometry.width), h: Math.round(node.geometry.height), }, }, ] }) } // ─── find ─── export type FindMode = | 'testid' | 'role' | 'type' | 'pressable' | 'interactive-targets' | 'visible' | 'text' export interface FindQuery { testId?: string | null role?: string | null type?: string | null text?: string | null pressable?: boolean visible?: boolean interactive?: boolean } // resolve a find query to its mode + structured selector. precedence matches // the CLI: testid → role → type → pressable → interactive → visible → text. // returns null when nothing was asked for (caller prints usage). export function resolveFindMode( q: FindQuery, ): { mode: FindMode; selector: SimSemanticSelector } | null { if (q.testId) { return { mode: 'testid', selector: { testID: q.testId } } } if (q.role) { return { mode: 'role', selector: { role: q.role } } } if (q.type) { return { mode: 'type', selector: { type: q.type } } } if (q.pressable) { return { mode: 'pressable', selector: { pressable: true } } } if (q.interactive) { return { mode: 'interactive-targets', selector: { pressable: true } } } if (q.visible) { return { mode: 'visible', selector: { visible: true } } } if (q.text) { return { mode: 'text', selector: { text: q.text } } } return null } // `find` — locate nodes by text / testID / role / type / predicate. returns // the raw bridge result (a node, a node array, or null) plus the resolved // mode; null when the query was empty. export async function inspectFind( bridge: InspectBridge, q: FindQuery, ): Promise<{ mode: FindMode; result: unknown } | null> { const resolved = resolveFindMode(q) if (!resolved) return null const result: unknown = await bridge.send({ type: 'query', query: { kind: 'find', selector: resolved.selector }, }) if (!isSemanticQueryResult(result)) return { mode: resolved.mode, result: [] } const nodes = result.nodes.map( (node): InteractiveNode => ({ type: node.type, nodeId: node.nodeId, ...(node.testID ? { testID: node.testID } : {}), ...(node.text ? { text: node.text } : {}), ...(node.pressable ? { pressable: true } : {}), ...(node.role ? { accessibilityRole: node.role } : {}), ...(node.label ? { accessibilityLabel: node.label } : {}), absolutePosition: { x: node.geometry.x, y: node.geometry.y }, layout: { width: node.geometry.width, height: node.geometry.height }, }), ) const resultValue = resolved.mode === 'testid' || resolved.mode === 'text' ? (nodes[0] ?? null) : nodes return { mode: resolved.mode, result: resultValue } } export interface InteractiveNode { type?: string text?: string testID?: string pressable?: boolean accessibilityRole?: string accessibilityLabel?: string absolutePosition?: { x: number; y: number } layout?: { width: number; height: number } [key: string]: unknown } // `find --interactive-targets` ranking. visible+pressable nodes scored by // identifying signal (testID > visible text > a11y label > role) and area, // with chrome / off-screen nodes de-prioritised. higher score = better // "next tap" candidate. shared so the CLI and the agent rank identically. export function rankInteractive(nodes: T[]): T[] { return [...nodes].sort((a, b) => scoreInteractive(b) - scoreInteractive(a)) } export function scoreInteractive(n: InteractiveNode): number { let score = 0 if (n.testID) score += 100 if (typeof n.text === 'string' && n.text.trim().length > 0) score += 60 if ( typeof n.accessibilityLabel === 'string' && n.accessibilityLabel.trim().length > 0 ) { score += 30 } if (n.accessibilityRole) score += 15 const w = n.layout?.width ?? 0 const h = n.layout?.height ?? 0 const area = w * h if (area >= 400 && area <= 60_000) score += 25 else if (area > 60_000) score -= 20 const y = n.absolutePosition?.y ?? 0 if (y < 0) score -= 30 return score } // the copy-paste `rnx do` tap command for an interactive node. export function tapCommandForNode(n: InteractiveNode): string { if (n.testID) return `rnx do tap-id ${shellEscape(n.testID)}` const text = typeof n.text === 'string' ? n.text.trim() : '' if (text.length > 0 && text.length <= 80) return `rnx do tap-text ${shellEscape(text)}` const x = Math.round(((n.absolutePosition?.x ?? 0) + (n.layout?.width ?? 0) / 2) * 10) / 10 const y = Math.round(((n.absolutePosition?.y ?? 0) + (n.layout?.height ?? 0) / 2) * 10) / 10 return `rnx do tap ${x} ${y}` } function shellEscape(value: string): string { if (/^[A-Za-z0-9_./@:-]+$/.test(value)) return value return `'${value.replace(/'/g, `'\\''`)}'` } // ─── wait ─── export interface WaitReadyStatus { ready: boolean elapsedMs: number nodes: number targets: number liveFrameActive: boolean liveFrameChannels: number liveFramePublishes: number flag: unknown loadingText: string externalReady: boolean | null externalStatus: string externalError: string suppressedEntryError: string errors: number // non-empty when the probe itself stopped reaching the sim (tab closed, // sim never connected) — the wait failed on the bridge, not on the app. bridgeError: string } export interface WaitReadyOptions { progressIntervalMs?: number onProgress?: (status: WaitReadyStatus) => void simId?: string } export function waitReadyReason( status: Pick< WaitReadyStatus, | 'externalError' | 'loadingText' | 'externalReady' | 'externalStatus' | 'flag' | 'targets' | 'suppressedEntryError' > & { bridgeError?: string }, ): string { if (status.bridgeError) { if ( status.bridgeError.includes(HEADLESS_TENANT_FETCH_HUNG_MARKER) || status.bridgeError.includes(PUBLIC_NO_NETWORK_MARKER) ) { return status.bridgeError } return `sim unreachable: ${status.bridgeError} — the target tab is closed or no sim is connected (check \`rnx list\`, reopen with \`rnx open\`)` } const waitingOnGuestFetch = status.externalReady === false && /https?:\/\//.test(status.externalStatus) const base = status.externalError ? `guest app errored: ${status.externalError}` : waitingOnGuestFetch ? `waiting on a guest fetch: ${status.externalStatus}` : status.loadingText ? `still showing "${status.loadingText}"` : status.externalReady === false ? 'guest app is still loading' : status.flag !== true && status.targets > 0 ? 'native content is rendered but the ready signal has not settled' : status.flag !== true ? 'guest app has not emitted sootsim:externalAppReady' : status.targets <= 0 ? 'ready flag emitted but no visible app content is inspectable yet' : 'node tree is still changing' // suppressed entry errors never gate readiness, but on a stall they are // often the actual diagnosis — surface them as a hint. return !status.externalError && status.suppressedEntryError ? `${base} (suppressed entry error: ${status.suppressedEntryError})` : base } // `wait ready` — block until the guest app bundle has mounted and painted. // the persistent `__sootsimExternalAppReady` flag is necessary but not enough: // under the shell renderer the flag can fire before the shell has materialized // the tenant tree, and the node count can then jump through placeholder // counts before the app has finished publishing its first real surface. require // visible inspectable content (or a larger non-placeholder tree) and a brief // stable count. the engine owns the full polling loop, so every caller pays // exactly one bridge request even for a cold guest bundle. export async function inspectWaitReady( bridge: InspectBridge, timeoutMs = 20_000, options: WaitReadyOptions = {}, ): Promise { const start = Date.now() let result: SemanticWaitResult try { result = await bridge.send( { type: 'waitFor', ...(options.simId ? { simId: options.simId } : {}), waitForOptions: { condition: { type: 'ready' }, timeoutMs }, }, { timeoutMs: timeoutMs + 1_000 }, ) } catch (error) { const bridgeError = error instanceof Error ? error.message : String(error) if ( bridgeError.startsWith('multiple sims are connected:') || bridgeError.startsWith('saved sim ') ) { throw error } result = { matched: false, elapsedMs: Date.now() - start, polls: 0, error: bridgeError, } } const probe = result.probe // a guest app that redboxes still paints: its ErrorBoundary is a real tree // with real nodes, so the engine's own readiness condition matches and every // caller reads exit 0 as "the app is up". an app that has errored is not // ready, so a guest error is disqualifying here rather than merely printed // on the failure path. `errors` (a console.error count) is deliberately not // used because apps can log errors benignly. `suppressedEntryError` remains // excluded because it never gates readiness by contract. const externalError = probe?.externalError ?? '' const status: WaitReadyStatus = { ready: result.matched && !externalError, elapsedMs: result.elapsedMs, nodes: probe?.nodes ?? 0, targets: probe?.targets ?? 0, liveFrameActive: probe?.liveFrameActive ?? false, liveFrameChannels: probe?.liveFrameChannels ?? 0, liveFramePublishes: probe?.liveFramePublishes ?? 0, flag: probe?.flag, loadingText: probe?.loadingText ?? '', externalReady: probe?.externalReady ?? null, externalStatus: probe?.externalStatus ?? '', externalError, suppressedEntryError: probe?.suppressedEntryError ?? '', errors: probe?.errors ?? 0, bridgeError: result.error ?? '', } if (!status.ready && options.onProgress) options.onProgress(status) return status } // `wait selector` — block until a node with the given testID is present and // laid out. the polling loop runs inside the evaluated code (one round-trip). export async function inspectWaitSelector( bridge: InspectBridge, testId: string, timeoutMs = 5000, opts: { gone?: boolean; structural?: boolean } = {}, ): Promise<{ found: boolean; node?: SimSemanticNode; elapsed: number }> { const gone = opts.gone === true const cleanId = testId.replace(/^#/, '') const result: SemanticWaitResult = await bridge.send( { type: 'waitFor', waitForOptions: { condition: { type: gone ? 'absent' : opts.structural === true ? 'mounted' : 'present', selector: { testID: cleanId }, }, timeoutMs, }, }, { timeoutMs: timeoutMs + 1_000 }, ) if (result?.error) throw new Error(result.error) if (result?.node && !isSimSemanticTree([result.node])) { throw new Error('waitFor returned an invalid semantic node') } return { found: result?.matched === true, ...(result?.node ? { node: result.node } : {}), elapsed: result?.elapsedMs ?? timeoutMs, } } // ─── console errors / warnings ─── // a captured console entry. `__sootsimConsole` records each console.error / // console.warn with its args and (for errors) a stack. `source` is set when // the entry came from the engine observability store (e.g. 'render-worker') // so the reader can tell page-realm errors from forwarded tenant-worker ones. export interface ConsoleEntry { timestamp: number args: unknown[] stack?: string source?: string [key: string]: unknown } // in-page merge of the two error/warning sources, evaluated in the preview // realm. `level` is 'error' | 'warn'. // // why two sources: `__sootsimConsole` (the ws-bridge ring buffer) only sees a // tenant render-worker error if the worker→host `{type:'log'}` re-emit makes it // through the host realm's monkey-patched `console.error` — and only when // ws-bridge's `setupConsoleCapture()` ran (gated on localhost/electron). The // engine observability store (`__sootsimObservability.logs`) captures the SAME // worker errors via two unconditional paths — the worker's own BroadcastChannel // capture and the host's `recordForwardedWorkerLog('render-worker', …)` — and // carries an explicit `source`/`level`. Reading only `__sootsimConsole` is how // a native render-worker crash (e.g. a tamagui "Missing theme" thrown by the // One root error boundary in the tenant worker) can be live and screaming in // the driver's console while the agent's `sootsim_errors` reports nothing. // Merge both, dedup, so worker + root-error-boundary failures always surface. function consoleEntriesEval(level: 'error' | 'warn', limit: number): string { const getter = level === 'error' ? 'getErrors' : 'getWarnings' return `(() => { const out = [] const seen = new Set() const norm = (s) => (typeof s === 'string' ? s : (() => { try { return JSON.stringify(s) } catch { return String(s) } })()) const key = (ts, args) => Math.round((ts || 0) / 250) + '|' + (Array.isArray(args) ? args.map(norm).join(' ') : norm(args)) const push = (e) => { if (!e) return const ts = typeof e.timestamp === 'number' ? e.timestamp : (typeof e.ts === 'number' ? e.ts : 0) const k = key(ts, e.args) if (seen.has(k)) return seen.add(k) out.push({ timestamp: ts, args: Array.isArray(e.args) ? e.args : [e.args], stack: e.stack || undefined, source: e.source || undefined }) } // page-realm ws-bridge buffer first (richest stacks for page-origin errors) try { for (const e of (window.__sootsimConsole?.${getter}(${limit}) || [])) push(e) } catch {} // engine observability store — the always-on render-worker / forwarded path try { const obs = window.__sootsimObservability const snap = obs && obs.logs && typeof obs.logs.getSnapshot === 'function' ? obs.logs.getSnapshot() : [] for (const e of snap) if (e && e.level === ${JSON.stringify(level)}) push(e) } catch {} return out.sort((a, b) => (a.timestamp || 0) - (b.timestamp || 0)).slice(-${limit}) })()` } // `get errors` — the most recent captured `console.error` entries, merged from // the ws-bridge buffer and the engine observability store so forwarded // tenant-worker errors (render-worker / root-error-boundary) are never missed. export async function inspectErrors( bridge: InspectBridge, limit = 20, ): Promise { const result = await bridge.send({ type: 'evaluate', code: consoleEntriesEval('error', limit), }) return Array.isArray(result) ? (result as ConsoleEntry[]) : [] } // `get warnings` — the most recent captured `console.warn` entries, merged from // both capture sources (see inspectErrors). export async function inspectWarnings( bridge: InspectBridge, limit = 20, ): Promise { const result = await bridge.send({ type: 'evaluate', code: consoleEntriesEval('warn', limit), }) return Array.isArray(result) ? (result as ConsoleEntry[]) : [] } // merged error/warning COUNT across both capture sources, evaluated in the // preview realm. used by the CLI's proactive "console: N errors" notice and // the `get state` diagnostics so the count agrees with what inspectErrors / // inspectWarnings actually return — otherwise a render-worker-only error would // read as `0 errors` and the agent would never be nudged to inspect it. export const MERGED_CONSOLE_COUNT_EVAL = `(() => { const norm = (s) => (typeof s === 'string' ? s : (() => { try { return JSON.stringify(s) } catch { return String(s) } })()) const key = (lvl, ts, args) => lvl + '|' + Math.round((ts || 0) / 250) + '|' + (Array.isArray(args) ? args.map(norm).join(' ') : norm(args)) const seen = new Set() let errors = 0 let warnings = 0 const add = (lvl, ts, args) => { if (lvl !== 'error' && lvl !== 'warn') return const k = key(lvl, ts, args) if (seen.has(k)) return seen.add(k) if (lvl === 'error') errors++ else warnings++ } try { const c = window.__sootsimConsole for (const e of (c?.getErrors?.(200) || [])) add('error', e?.timestamp ?? e?.ts, e?.args) for (const e of (c?.getWarnings?.(200) || [])) add('warn', e?.timestamp ?? e?.ts, e?.args) } catch {} try { const obs = window.__sootsimObservability const snap = obs && obs.logs && typeof obs.logs.getSnapshot === 'function' ? obs.logs.getSnapshot() : [] for (const e of snap) if (e) add(e.level, e.ts, e.args) } catch {} return { errors, warnings, total: errors + warnings } })()` // `get errors clear` — drop BOTH captured buffers so a follow-up read starts // clean (the ws-bridge ring buffer and the engine observability log store). export async function clearConsole(bridge: InspectBridge): Promise { await bridge.send({ type: 'evaluate', code: 'window.__sootsimConsole?.clear(); window.__sootsimObservability?.logs?.clear?.(); "cleared"', }) } // ─── network requests ─── // `get requests` — the captured fetch / XHR entries. `failed` restricts to // unsuccessful / errored requests (the CLI's default); pass `failed: false` for // every request. backed by the test bridge's request observability store. export async function inspectRequests( bridge: InspectBridge, opts: { failed?: boolean; limit?: number } = {}, ): Promise { const limit = opts.limit ?? 20 const method = opts.failed === false ? 'getRequests' : 'getFailedRequests' const result = await bridge.send({ type: 'call', path: `__sootsimTest.${method}`, args: [limit], }) return Array.isArray(result) ? (result as SootSimRequestEntry[]) : [] } // `get requests clear` — drop the captured request buffer. export async function clearRequests(bridge: InspectBridge): Promise { await bridge.send({ type: 'call', path: '__sootsimTest.clearRequests', args: [] }) } // ─── scroll state ─── export interface ScrollState { scrollOffsetX?: number scrollOffsetY?: number offsetX?: number offsetY?: number [k: string]: unknown } // the scroll state of whichever scrollable sits under a screen coordinate. // used to verify a swipe / scroll actually moved content — read it before // and after the gesture and compare the offsets. export async function inspectScrollStateAt( bridge: InspectBridge, x: number, y: number, ): Promise { const result = await bridge.send({ type: 'call', path: '__sootsimTest.getScrollStateAt', args: [x, y], }) if (!result || typeof result !== 'object') return null const state = result as ScrollState return { ...state, scrollOffsetX: typeof state.scrollOffsetX === 'number' ? state.scrollOffsetX : state.offsetX, scrollOffsetY: typeof state.scrollOffsetY === 'number' ? state.scrollOffsetY : state.offsetY, } } // ─── console logs ─── export type LogLevel = 'log' | 'info' | 'warn' | 'error' | 'debug' export interface LogEntry { id: string source: string level: LogLevel ts: number args: string[] stack: string | null } export interface LogFilterOptions { level?: ReadonlySet | null filter?: string | null // engine-internal `[sootsim]` bootstrap logs are hidden unless this is set. showInternal?: boolean } // `logs` — the full captured console snapshot (log / info / warn / error / // debug) from the engine observability store. when the observability bridge has // not installed yet, return an empty snapshot so validation tools don't turn a // missing diagnostics buffer into the app failure being diagnosed. export async function inspectLogs(bridge: InspectBridge): Promise { if (bridge.plane === 'cloud') { const state: unknown = await bridge.send({ type: 'state', stateOptions: { screenshot: false, tree: false, route: false, errors: false, logs: true, logLimit: 100, }, }) const recentLogs = state && typeof state === 'object' && Array.isArray(Reflect.get(state, 'recentLogs')) ? Reflect.get(state, 'recentLogs') : [] return recentLogs.flatMap((entry: unknown, index: number) => { if (!entry || typeof entry !== 'object') return [] const text = Reflect.get(entry, 'text') const rawLevel = Reflect.get(entry, 'level') if (typeof text !== 'string') return [] const level: LogLevel = rawLevel === 'log' || rawLevel === 'info' || rawLevel === 'warn' || rawLevel === 'error' || rawLevel === 'debug' ? rawLevel : 'error' const at = Reflect.get(entry, 'at') const ts = typeof at === 'number' && Number.isFinite(at) ? at : 0 return [ { id: `cloud-${ts}-${index}`, source: 'rnx-cloud', level, ts, args: [text], stack: null, }, ] }) } const res = (await bridge.send({ type: 'evaluate', code: `(() => { const obs = window.__sootsimObservability; if (!obs) return { ok: false }; return { ok: true, entries: obs.logs.getSnapshot() }; })()`, })) as { ok: boolean; entries?: LogEntry[] } | null if (!res || !res.ok) return [] return res.entries ?? [] } // `logs clear` — drop the engine's captured log buffer. export async function clearLogs(bridge: InspectBridge): Promise { await bridge.send({ type: 'evaluate', code: 'window.__sootsimObservability?.logs.clear(); "cleared"', }) } function isForwardedWorkerLogTwin(a: LogEntry, b: LogEntry): boolean { if (a.source === b.source) return false if (a.level !== b.level) return false if (Math.abs(a.ts - b.ts) > 1000) return false if (a.args.length !== b.args.length) return false const sources = new Set([a.source, b.source]) if (!sources.has('sootsim-worker')) return false if (!sources.has('render-worker') && !sources.has('forwarded-render-worker')) { return false } return a.args.every((arg, index) => arg === b.args[index]) } // shared log filtering — engine-internal `[sootsim]` noise hidden by default, // optional level set + message substring. used by the CLI `logs` verb and the // agent so both see the same filtered view. export function filterLogEntries( entries: LogEntry[], opts: LogFilterOptions = {}, ): LogEntry[] { let out: LogEntry[] = [] for (const entry of entries) { if (out.some((candidate) => isForwardedWorkerLogTwin(candidate, entry))) { continue } out.push(entry) } if (!opts.showInternal) { out = out.filter((e) => { const first = e.args[0] return !( typeof first === 'string' && /^\[(?:rnx|sootsim)(?:\s[^\]]*)?\]/.test(first) ) }) } if (opts.level) out = out.filter((e) => opts.level!.has(e.level)) if (opts.filter) { const lf = opts.filter.toLowerCase() out = out.filter((e) => e.args.join(' ').toLowerCase().includes(lf)) } return out } // ─── timeline / what-happened ─── // `what-happened --summary` — counts by kind since the query window. export async function inspectTimelineSummary( bridge: InspectBridge, query: SootSimTimelineQuery, ): Promise { return (await bridge.send({ type: 'call', path: 'SootSim.bridges.timeline.summary', args: [query], })) as SootSimTimelineSummary } // `what-happened` — the recent timeline events for the query window. export async function inspectTimelineRecent( bridge: InspectBridge, query: SootSimTimelineQuery, ): Promise { return (await bridge.send({ type: 'call', path: 'SootSim.bridges.timeline.recent', args: [query], })) as SootSimTimelineQueryResult } // a blocking native-UI surface (iOS Alert / ActionSheet) currently presented // by the shell worker. `label` is the human word for the CLI note; `title` is // the surface's title text when it has one. export interface OpenNativeUISurface { label: string title: string | null } // detect blocking native-UI surfaces that are currently open, computed from the // timeline. the shell native-UI provider records a `show` then a `resolve` // event per surface (both DEFAULT_ON kinds, so no `timeline start` is needed), // so the latest event of each kind tells whether it's still up. the CLI surfaces // this because these surfaces live in the shell worker — invisible to `describe` // (which reads the tenant tree) — yet an open modal silently swallows every // tap/drag/type meant for the app underneath, which reads as "the action did // nothing." returns [] on any read error; this is an advisory note, never fatal. export async function detectOpenNativeUI( bridge: InspectBridge, ): Promise { let events: SootSimTimelineEvent[] try { const res = await inspectTimelineRecent(bridge, { kinds: ['alert', 'actionsheet'], limit: 80, }) events = res.events } catch { return [] } // events arrive time-ordered; keep the latest event per kind. const latest = new Map() for (const ev of events) latest.set(ev.kind, ev) const out: OpenNativeUISurface[] = [] const check = (kind: string, label: string) => { const ev = latest.get(kind) if (!ev || !ev.data || typeof ev.data !== 'object') return // timeline payloads are plain JSON records (see formatTimelinePayload) const d = ev.data as Record if (d.phase !== 'show') return out.push({ label, title: typeof d.title === 'string' ? d.title : null }) } check('alert', 'Alert') check('actionsheet', 'ActionSheet') return out } // advance a caller's cursor past the events it has consumed, so the next // `what-happened` for the same cursor key shows only newer events. export async function inspectTimelineAdvanceCursor( bridge: InspectBridge, cursorKey: string, watermark: number, ): Promise { await bridge.send({ type: 'call', path: 'SootSim.bridges.timeline.cursorAdvance', args: [cursorKey, watermark], }) } // ─── keyboard ─── // `__sootsimKeyboard.getLayout()` returns the shell's own snapshot, so the // engine's contract is the type. a copy here drifts from it silently. export type KeyboardState = KeyboardLayoutSnapshot // `get keyboard` — the live iOS keyboard state from `__sootsimKeyboard`, // populated by shell-worker `keyboard.layout` broadcasts. returns the raw // payload or `{ error }` when the keyboard bridge has not installed yet. export async function inspectKeyboard( bridge: InspectBridge, ): Promise { const result = (await bridge.send({ type: 'evaluate', code: `(() => { const kb = window.__sootsimKeyboard if (!kb || typeof kb.getLayout !== 'function') { return { error: 'keyboard bridge getLayout() not available' } } return kb.getLayout() })()`, })) as KeyboardState | { error: string } return result ?? { error: 'keyboard bridge returned no result' } } // ─── shell state ─── // true when a `SootSim.bridges.mainShell.*` call failed because the shell // bridge has not installed yet (boot race) — distinct from a real error. export function isShellCommandUnavailable(error: unknown): boolean { const message = error instanceof Error ? error.message : String(error) return ( message.includes('call target not found: SootSim.bridges.mainShell') || message.includes('test bridge unavailable before app-in-worker boot') ) } // read `mainShell.getState()`, retrying past the boot race for up to // `readyTimeoutMs`. shared by the CLI's inspect verbs and the agent. export async function getShellState( bridge: InspectBridge, readyTimeoutMs = 0, ): Promise | null> { const deadline = Date.now() + Math.max(0, readyTimeoutMs) while (true) { try { return (await bridge.send({ type: 'call', path: 'SootSim.bridges.mainShell.getState', args: [], })) as Record | null } catch (error) { if (!isShellCommandUnavailable(error) || Date.now() >= deadline) throw error await new Promise((r) => setTimeout(r, 50)) } } } // ─── screens ─── export interface ScreenEntry { id: string routeName?: string isActive: boolean headerHeight: number largeTitleState?: string } export interface NavSnapshot { screens: ScreenEntry[] activeScreenId: string | null transitionPhase: string activeTransitionCount: number activeHeaderHeight: number activeLargeTitleState?: string } export interface ScreensReport { shell: Record | null nav: NavSnapshot | null route: SootSimRouteInfo | null keyboard: { visible: boolean mode?: string spec?: { keyboardType?: string; returnKeyType?: string } | null } | null } // `get screens` — the navigation stack + shell state + keyboard, a "what // screen am I on?" dashboard. nav + keyboard are read in one evaluate so the // snapshot is consistent; shell state is a best-effort separate call. export async function inspectScreens(bridge: InspectBridge): Promise { const payload = (await bridge.send({ type: 'evaluate', code: `(async () => { const test = window.__sootsimTest const kb = window.__sootsimKeyboard const navSnap = test && typeof test.getNavigationSnapshot === 'function' ? await test.getNavigationSnapshot() : null const route = test && typeof test.getRouteInfo === 'function' ? await test.getRouteInfo() : null const keyboard = kb && typeof kb.getLayout === 'function' ? (() => { const layout = kb.getLayout() return layout ? { visible: layout.visible, mode: layout.mode, spec: layout.spec ? { keyboardType: layout.spec.keyboardType, returnKeyType: layout.spec.returnKeyType, } : null, } : null })() : null return { nav: navSnap, route, keyboard } })()`, })) as { nav: NavSnapshot | null route: SootSimRouteInfo | null keyboard: ScreensReport['keyboard'] } | null const shell = await getShellState(bridge, 500).catch(() => null) return { shell, nav: payload?.nav ?? null, route: payload?.route ?? null, keyboard: payload?.keyboard ?? null, } } // ─── debug channels ─── // the `window.__sootsimDebug` channels the engine instruments. `all` is a // meta-target accepted by enable/disable. export const DEBUG_CHANNELS = [ 'portals', 'sheets', 'layout', 'onlayout', 'animated', 'render', 'touch', 'yoga', ] as const export type DebugChannel = (typeof DEBUG_CHANNELS)[number] // `debug status` — which debug channels are currently enabled + their counts. export async function inspectDebugStatus(bridge: InspectBridge): Promise { return bridge.send({ type: 'evaluate', code: 'window.__sootsimDebug.status()' }) } // `debug flags` — the engine's debug feature flags. export async function inspectDebugFlags(bridge: InspectBridge): Promise { return bridge.send({ type: 'evaluate', code: 'window.__sootsimDebug.flags()' }) } // `debug find sheets|portals` — locate all live sheet / portal nodes. // routed through the test bridge (not the page __sootsimDebug global) so the // walk runs against the worker's live tree — the page-local tree is empty // under the worker renderer. export async function inspectDebugFind( bridge: InspectBridge, target: 'sheets' | 'portals' | 'boundaries', ): Promise { return bridge.send({ type: 'evaluate', code: `window.__sootsimTest.debugFind(${JSON.stringify(target)})`, }) } // debug buffers belong to their execution context. tenant calls use the // existing test bridge; host calls keep shell/compositor flag forwarding intact. export async function callDebugBridge( bridge: InspectBridge, expression: string, host = false, ): Promise { const code = `globalThis.__sootsimDebug.${expression}` if (host) return bridge.send({ type: 'evaluate', code }) return bridge.send({ type: 'evaluate', code: `(async () => { const result = await window.__sootsimTest.evalInTenant(${JSON.stringify(code)}) if (!result.ok) throw new Error(result.error) return result.value })()`, }) } // `debug recent` reads tenant events; --host explicitly selects the page buffer. export async function inspectDebugRecent( bridge: InspectBridge, channel?: string, limit = 50, host = false, ): Promise { return callDebugBridge( bridge, `recent(${channel && channel !== 'all' ? JSON.stringify(channel) : 'undefined'}, ${limit})`, host, ) } // channel controls reach the host (and its shell/compositor subscribers) plus // the active tenant. await the tenant reply before reporting success. export async function setDebugChannels( bridge: InspectBridge, action: 'enable' | 'disable', channels: string[], host = false, ): Promise { const args = channels.length > 0 ? channels.map((c) => JSON.stringify(c)).join(', ') : action === 'disable' ? "'all'" : '' const expression = `${action}(${args})` const active = await callDebugBridge(bridge, expression, true) if (host) return active return callDebugBridge(bridge, expression) } // ─── memory ─── export interface ImageLoaderStats { cacheEntries: number cachePixelBytes: number cachePixelBudget: number cacheMaxEntries: number pendingFetches: number pendingBytes: number failedUris: number snapshots: number liveFrames: number } export interface WorkerHeapStats { usedJSHeapSize: number totalJSHeapSize: number jsHeapSizeLimit: number } export interface EngineObjectCounts { pictures: number rasterImages: number paragraphs: number yogaNodes: number } export interface WorkerMemorySample { // total nodes ever registered in the worker's handleToNode map — a // monotonically climbing value while the on-screen tree is stable means // discarded subtrees are being retained. nodesRegistered: number | null // registered nodes not reachable from a root container, by count and type nodesDetached: number | null detachedTypes: Record | null objects: EngineObjectCounts | null activeNativeAnimations: number | null imageLoader: ImageLoaderStats | null workerHeap: WorkerHeapStats | null // canvaskit's wasm linear memory in this worker. every worker that loads // canvaskit allocates its own, no js-heap counter includes it, and on webkit // (where performance.memory does not exist) it is the only number available. wasmHeapBytes: number | null } export interface MemoryReport { tenant: WorkerMemorySample | null shell: WorkerMemorySample | null compositor: WorkerMemorySample | null hostHeap: WorkerHeapStats | null } // `get memory` / `debug memory` — per-worker object counts + image-loader // cache + JS heaps. tenant numbers ride the queryStats memory block; the // shell worker answers via __sootsimShellMemory, and the compositor — which // paints home + app:one + app:two, and therefore holds every app-surface // picture and raster image — answers through its stats channel. // performance.memory is chrome-only, null on webkit. export async function inspectMemory(bridge: InspectBridge): Promise { if (bridge.plane === 'cloud') { const value: unknown = await bridge.send({ type: 'memory' }) const rawWasmHeapBytes = typeof value === 'object' && value !== null ? Reflect.get(value, 'wasmHeapBytes') : null const wasmHeapBytes = typeof rawWasmHeapBytes === 'number' ? rawWasmHeapBytes : null return { tenant: { nodesRegistered: null, nodesDetached: null, detachedTypes: null, objects: null, activeNativeAnimations: null, imageLoader: null, workerHeap: null, wasmHeapBytes, }, shell: null, compositor: null, hostHeap: null, } } const result = (await bridge.send({ type: 'evaluate', code: `(async () => { const host = window.__sootsimRenderHost const stats = host?.queryStats ? await host.queryStats() : null const shellRaw = window.__sootsimShellMemory ? await window.__sootsimShellMemory() : null const shell = shellRaw && !shellRaw.error ? shellRaw : null // the compositor paints home + app:one + app:two, so every app-surface // picture and raster image is counted here and nowhere else. let compositor = null try { const cs = await window.__sootsimCompositor?.getStats?.(false, true) compositor = cs?.memory ?? null } catch {} const normalize = (m) => m ? { nodesRegistered: m.nodesRegistered ?? null, nodesDetached: m.nodesDetached ?? null, detachedTypes: m.detachedTypes ?? null, objects: m.objects ?? null, activeNativeAnimations: m.activeNativeAnimations ?? null, imageLoader: m.imageLoader ?? null, workerHeap: m.workerHeap ?? null, wasmHeapBytes: m.wasmHeapBytes ?? null, } : null const hostMem = performance.memory ? { usedJSHeapSize: performance.memory.usedJSHeapSize, totalJSHeapSize: performance.memory.totalJSHeapSize, jsHeapSizeLimit: performance.memory.jsHeapSizeLimit, } : null return { tenant: normalize(stats?.memory), shell: normalize(shell), compositor: normalize(compositor), hostHeap: hostMem, } })()`, })) as MemoryReport | null return result ?? { tenant: null, shell: null, compositor: null, hostHeap: null } } // one-line counts, ordered by descriptive priority. shared by the CLI's // `what-happened --summary` and the agent's sootsim_what_happened. export function formatTimelineSummary(summary: SootSimTimelineSummary): string { if (summary.total === 0) return 'nothing recorded' const parts: string[] = [] const ORDER = [ 'error', 'warning', 'console', 'fetch', 'toast', 'alert', 'actionsheet', 'picker', 'notification', 'screen', 'route', 'keyboard', 'app-launch', 'shell', 'scroll', 'gesture', 'text-input', 'react-commit', 'animation', 'reanimated', ] const seen = new Set() for (const k of ORDER) { const n = summary.byKind[k] if (n) { parts.push(`${n} ${k}${n === 1 ? '' : 's'}`) seen.add(k) } } for (const [k, n] of Object.entries(summary.byKind)) { if (!seen.has(k) && n) parts.push(`${n} ${k}${n === 1 ? '' : 's'}`) } return parts.join(' · ') } // high-frequency render/layout/scroll kinds that drown the meaningful // events. both the `what-happened` CLI and the sootsim_what_happened agent // tool hide these by default so a single tap doesn't get buried under 1000+ // react-commit rows. callers opt back in explicitly. export const NOISY_TIMELINE_KINDS: ReadonlySet = new Set([ 'react-commit', 'layout', 'scroll', ]) function formatTimelineRelativeTime(t: number, anchor: number | null): string { if (anchor === null) return new Date(t).toLocaleTimeString() const dt = (t - anchor) / 1000 const sign = dt >= 0 ? '+' : '' return `${sign}${dt.toFixed(2)}s` } // payload rendering per timeline-event kind. one exhaustive switch over // SootSimTimelineKind so adding a kind forces a formatter decision rather // than silently leaving it JSON-dumped. shared by the `what-happened` CLI and // the sootsim_what_happened agent tool — one renderer, two transports. export function formatTimelinePayload( kind: SootSimTimelineKind, d: Record, ): string { switch (kind) { case 'app-launch': return d.phase === 'launch' ? `launch ${d.appName ?? d.toAppId ?? ''}` : `dismiss ${d.appName ?? d.fromAppId ?? ''} → ${d.toAppId ?? ''}` case 'toast': return `"${d.text ?? ''}"${d.durationMs ? ` (${d.durationMs}ms)` : ''}` case 'keyboard': return `${d.phase ?? '?'}${d.heightPx ? ` h=${d.heightPx}` : ''}${d.mode ? ` ${d.mode}` : ''}` case 'screen': return `${d.phase ?? '?'} ${d.name ?? d.activeName ?? ''}` case 'flow-step': return `${d.status ?? '?'} ${d.stepName ?? 'step'}${d.durationMs ? ` ${d.durationMs}ms` : ''}` case 'route': return `${d.phase ?? '?'} ${d.path ?? d.pathname ?? ''}` case 'alert': case 'actionsheet': case 'picker': return `${d.phase ?? '?'} ${d.title ?? d.message ?? ''}` case 'notification': return `${d.title ?? ''}${d.body ? ` — ${d.body}` : ''}` case 'fetch': return `${d.method ?? 'GET'} ${d.url ?? ''}${d.status ? ` -> ${d.status}` : ''}` case 'console': case 'console-log': return `${d.level ?? 'log'}: ${(d.message ?? '').toString().slice(0, 120)}` case 'shell': return `${d.event ?? d.type ?? d.phase ?? ''}` case 'scroll': return `${d.phase ?? '?'} ${d.target ?? ''}` case 'gesture': return `${d.phase ?? '?'} ${d.type ?? ''}` case 'text-input': return `${d.phase ?? '?'}${d.value !== undefined ? ` "${String(d.value).slice(0, 40)}"` : ''}` case 'layout': return `${d.kind ?? '?'} ${d.testID ?? d.type ?? ''}${ d.skipped ? ` skipped:${d.reason ?? 'unknown'}` : '' }` case 'react-commit': { const slowest = d.slowest as | { displayName?: unknown; durationMs?: unknown } | null | undefined return `${d.fiberCount ?? '?'} fibers ${d.durationMs ?? '?'}ms${ slowest?.displayName ? ` · ${slowest.displayName} ${slowest.durationMs ?? '?'}ms` : '' }` } case 'frame': return `${d.totalMs ?? '?'}ms${d.renderMs ? ` · render ${d.renderMs}ms` : ''}${ d.layoutMs ? ` · layout ${d.layoutMs}ms` : '' }${d.copyMs ? ` · copy ${d.copyMs}ms` : ''}` case 'reanimated': case 'animation': return `${d.kind ?? ''} ${d.target ?? ''}${d.durationMs ? ` ${d.durationMs}ms` : ''}` } return '' } // one compact line per event: ` +0.42s shell [fetch] GET /api -> 200`. // pass anchor = first event's `t` for relative times, or null for wall-clock. export function formatTimelineEvent( event: SootSimTimelineEvent, anchor: number | null, ): string { const ts = formatTimelineRelativeTime(event.t, anchor).padStart(8) const ctx = event.context.padEnd(6) const kind = `[${event.kind}]`.padEnd(15) const d = event.data as Record | null const payload = d && typeof d === 'object' ? formatTimelinePayload(event.kind, d) : '' return ` ${ts} ${ctx} ${kind} ${payload}` } // render an event list as compact lines, anchored to the first event so times // read +0.00s, +0.13s, … . the agent tool and CLI both call this rather than // JSON-dumping the events array. export function formatTimelineEvents(events: readonly SootSimTimelineEvent[]): string { if (!events.length) return '' const anchor = events[0]?.t ?? null return events.map((e) => formatTimelineEvent(e, anchor)).join('\n') }