// public, runtime-neutral screen capture contract. semantic evidence and pixel // evidence are separate because neither one proves the other. export const RNX_SCREEN_CAPTURE_VERSION = 1 as const export interface RnxCaptureRect { x: number y: number width: number height: number } export interface RnxCaptureAccessibilityState { disabled: boolean selected: boolean checked: boolean | 'mixed' busy: boolean expanded: boolean } export interface RnxCaptureAccessibilityValue { min?: number max?: number now?: number text?: string } export interface RnxCaptureAccessibility { accessible: boolean label: string | null role: string | null hint: string | null liveRegion: 'none' | 'polite' | 'assertive' | null state: RnxCaptureAccessibilityState | null value: RnxCaptureAccessibilityValue | null } export interface RnxCaptureInteractionState { pressed: boolean pressable: boolean textInput: boolean textSelection: { start: number | null; end: number | null } | null secureTextEntry: boolean placeholder: string | null scrollOffset: { x: number; y: number } zoomScale: number | null nativePinchEnabled: boolean contentOffsetProp: { x?: number; y?: number } | null decelerationRateProp: unknown strokeDashoffsetProp: number | null } export interface RnxCaptureStyleTokenBinding { token: string theme: string } export interface RnxCaptureTextSegment { text: string color?: string | number fontFamily?: string fontSize?: number fontStyle?: string fontWeight?: string | number letterSpacing?: number textDecorationLine?: string } export interface RnxCaptureTextLine { text: string x: number y: number width: number height: number baseline: number segments?: RnxCaptureTextSegment[] } export interface RnxCaptureSource { location?: string file?: string componentName?: string componentStack?: string[] react?: { component?: string owners?: string[] source?: string key?: string hasRef?: boolean hooks?: { total: number counts: Record } contexts?: string[] lastCommit?: { ageMs: number causes: string[] propDiffs?: string[] stateChanged?: boolean actualDurationMs?: number } } } export interface RnxCaptureNode { nodeId: number parentNodeId: number | null childNodeIds: number[] paintIndex: number type: string id: string | null testID: string | null text: string | null childCount: number accessibility: RnxCaptureAccessibility interaction: RnxCaptureInteractionState layout: { local: RnxCaptureRect frame: RnxCaptureRect | null absolutePosition: { x: number; y: number } visibleFrame: RnxCaptureRect | null } style: { resolved: Record authored: Record computed: Record tokenBindings: Record } source: RnxCaptureSource props: Record textLines?: RnxCaptureTextLine[] /** additional generic inspect fields preserved for tooling adapters */ inspection: Record } export interface RnxCaptureTree { viewport: { width: number; height: number } rootNodeId: number nodes: RnxCaptureNode[] /** reconciler commit shared by semantic, layout, and visual evidence */ commitRevision?: number } export interface RnxCaptureVisualEvidence { kind: 'png' /** names the runtime that produced these pixels without claiming visibility */ producer: 'headless-cpu-app-surface' | 'browser-compositor-app-surface' dataUri: string pixelWidth: number pixelHeight: number } export interface RnxCaptureRegionRequest { key: string /** host node whose own subtree is rendered into this transparent atlas region */ nodeId: number frame: RnxCaptureRect } export interface RnxRasterRegionTree { width: number height: number nodes: readonly { nodeId: number parentNodeId: number | null type: string text?: string | null style: Record layout: RnxCaptureRect frame: RnxCaptureRect | null absolutePosition: { x: number; y: number } visibleFrame?: RnxCaptureRect | null }[] } export function selectRnxRasterNodeIds(tree: RnxRasterRegionTree): Set { const nodesById = new Map(tree.nodes.map((node) => [node.nodeId, node])) const unsupported = new Set() for (const node of tree.nodes) { const visibleFrame = node.visibleFrame ?? node.frame if (!visibleFrame || visibleFrame.width <= 0 || visibleFrame.height <= 0) continue if ( node.type === 'masked-view' || (node.type === 'text' && typeof node.text === 'string' && (/\p{Emoji_Presentation}/u.test(node.text) || /\uFE0F|\u200D|\u20E3|\p{Emoji_Modifier}|\p{Regional_Indicator}/u.test( node.text, ))) || node.style._liquidGlass !== undefined || node.style._liquidGlassEdge !== undefined || node.style._liquidGlassMaterialTint !== undefined ) { unsupported.add(node.nodeId) } } for (const nodeId of [...unsupported]) { let parentNodeId = nodesById.get(nodeId)?.parentNodeId ?? null while (parentNodeId !== null) { if (unsupported.has(parentNodeId)) { unsupported.delete(nodeId) break } parentNodeId = nodesById.get(parentNodeId)?.parentNodeId ?? null } } const screenArea = tree.width * tree.height const maxLayerArea = screenArea * 0.25 const maxAtlasArea = screenArea * 0.5 const candidates = [...unsupported] .map((nodeId) => { const node = nodesById.get(nodeId) const frame = node?.visibleFrame ?? node?.frame return frame ? { nodeId, area: frame.width * frame.height } : null }) .filter( (candidate): candidate is { nodeId: number; area: number } => candidate !== null && candidate.area <= maxLayerArea, ) .sort((left, right) => left.area - right.area || left.nodeId - right.nodeId) const selected = new Set() let atlasArea = 0 for (const candidate of candidates) { if (atlasArea + candidate.area > maxAtlasArea) continue selected.add(candidate.nodeId) atlasArea += candidate.area } return selected } export function rnxRasterFrames( tree: RnxRasterRegionTree, nodeIds: ReadonlySet, ): Map { const frames = new Map() for (const node of tree.nodes) { if (!nodeIds.has(node.nodeId)) continue const frame = node.frame ?? { x: node.absolutePosition.x, y: node.absolutePosition.y, width: node.layout.width, height: node.layout.height, } const liquidGlass = node.style._liquidGlass const capturePadding = liquidGlass && typeof liquidGlass === 'object' && typeof Reflect.get(liquidGlass, 'capturePadding') === 'number' ? Reflect.get(liquidGlass, 'capturePadding') : 0 const shadowRadius = typeof node.style.shadowRadius === 'number' ? node.style.shadowRadius : 0 const shadowOffset = node.style.shadowOffset const shadowOffsetX = shadowOffset && typeof shadowOffset === 'object' && typeof Reflect.get(shadowOffset, 'width') === 'number' ? Reflect.get(shadowOffset, 'width') : 0 const shadowOffsetY = shadowOffset && typeof shadowOffset === 'object' && typeof Reflect.get(shadowOffset, 'height') === 'number' ? Reflect.get(shadowOffset, 'height') : 0 const left = Math.max( 0, frame.x - Math.max(capturePadding, shadowRadius - shadowOffsetX), ) const top = Math.max( 0, frame.y - Math.max(capturePadding, shadowRadius - shadowOffsetY), ) const right = Math.min( tree.width, frame.x + frame.width + Math.max(capturePadding, shadowRadius + shadowOffsetX), ) const bottom = Math.min( tree.height, frame.y + frame.height + Math.max(capturePadding, shadowRadius + shadowOffsetY), ) if (right > left && bottom > top) { frames.set(node.nodeId, { x: left, y: top, width: right - left, height: bottom - top, }) } } return frames } export function rnxCaptureRasterRegions( capture: RnxScreenCapture, ): RnxCaptureRegionRequest[] { const tree: RnxRasterRegionTree = { width: capture.tree.viewport.width, height: capture.tree.viewport.height, nodes: capture.tree.nodes.map((node) => ({ nodeId: node.nodeId, parentNodeId: node.parentNodeId, type: node.type, text: node.text, style: node.style.resolved, layout: node.layout.local, frame: node.layout.frame, absolutePosition: node.layout.absolutePosition, visibleFrame: node.layout.visibleFrame, })), } const nodeIds = selectRnxRasterNodeIds(tree) const frames = rnxRasterFrames(tree, nodeIds) return [...nodeIds].flatMap((nodeId) => { const frame = frames.get(nodeId) return frame ? [{ key: String(nodeId), nodeId, frame }] : [] }) } /** a region request snapped to the surface pixel grid and placed in the atlas */ export interface RnxPackedCaptureRegion extends RnxCaptureRegionRequest { /** top-left of the region on the app surface, in pixels */ sourceX: number sourceY: number pixelWidth: number pixelHeight: number /** top-left of the region inside the atlas, in pixels */ atlasX: number atlasY: number } export interface RnxPackedCaptureAtlas { /** logical atlas dimensions; pixel dimensions are these times the pixel ratio */ width: number height: number pixelWidth: number pixelHeight: number regions: RnxPackedCaptureRegion[] /** the wire regions a producer publishes beside the encoded atlas */ atlasRegions: RnxCaptureAtlasRegion[] } // the bounded selective-layer atlas every producer ships: regions snapped to // whole surface pixels, shelf-packed into the smallest of a few candidate // widths, and mapped back to logical atlas coordinates. one packer for the // headless renderer (which draws isolated subtrees into the slots) and the // browser producer (which crops them out of a screen-sized screenshot), so a // consumer never learns which one made the artifact from the layout. export function packRnxCaptureRegions( regions: readonly RnxCaptureRegionRequest[], pixelRatio: number, surface: { pixelWidth: number; pixelHeight: number }, ): RnxPackedCaptureAtlas { const physical = regions.map((region) => { const left = Math.floor(region.frame.x * pixelRatio) const top = Math.floor(region.frame.y * pixelRatio) const right = Math.ceil((region.frame.x + region.frame.width) * pixelRatio) const bottom = Math.ceil((region.frame.y + region.frame.height) * pixelRatio) if ( left < 0 || top < 0 || right > surface.pixelWidth || bottom > surface.pixelHeight || right <= left || bottom <= top ) { throw new Error(`capture region ${region.key} is outside the app surface`) } return { key: region.key, nodeId: region.nodeId, frame: { x: left / pixelRatio, y: top / pixelRatio, width: (right - left) / pixelRatio, height: (bottom - top) / pixelRatio, }, sourceX: left, sourceY: top, pixelWidth: right - left, pixelHeight: bottom - top, } }) const largestWidth = Math.max(...physical.map((region) => region.pixelWidth)) const totalArea = physical.reduce( (area, region) => area + region.pixelWidth * region.pixelHeight, 0, ) if (totalArea > 4_194_304) { throw new Error('capture region pixels exceed the atlas budget') } const candidates = [ largestWidth, Math.max(largestWidth, Math.ceil(Math.sqrt(totalArea))), Math.max(largestWidth, surface.pixelWidth), 4_096, ] const ordered = [...physical].sort( (left, right) => right.pixelHeight - left.pixelHeight || right.pixelWidth - left.pixelWidth || left.key.localeCompare(right.key), ) let best: { width: number; height: number; regions: RnxPackedCaptureRegion[] } | null = null for (const candidate of [...new Set(candidates)]) { if (candidate > 4_096) continue let x = 0 let y = 0 let rowHeight = 0 let usedWidth = 0 const packed: RnxPackedCaptureRegion[] = [] for (const region of ordered) { if (x > 0 && x + region.pixelWidth > candidate) { y += rowHeight x = 0 rowHeight = 0 } packed.push({ ...region, atlasX: x, atlasY: y }) x += region.pixelWidth rowHeight = Math.max(rowHeight, region.pixelHeight) usedWidth = Math.max(usedWidth, x) } const height = y + rowHeight if (height > 4_096 || usedWidth * height > 4_194_304) continue const result = { width: usedWidth, height, regions: packed } if (!best || result.width * result.height < best.width * best.height) best = result } if (!best) throw new Error('capture regions cannot fit in one bounded atlas') const width = best.width / pixelRatio const height = best.height / pixelRatio return { width, height, pixelWidth: best.width, pixelHeight: best.height, regions: best.regions, atlasRegions: best.regions.map((region) => ({ key: region.key, nodeId: region.nodeId, frame: region.frame, // a region flush with the far edge is anchored to the logical edge so // its source never runs past the atlas by a rounding hair source: { x: region.atlasX + region.pixelWidth === best.width ? width - region.frame.width : region.atlasX / pixelRatio, y: region.atlasY + region.pixelHeight === best.height ? height - region.frame.height : region.atlasY / pixelRatio, width: region.frame.width, height: region.frame.height, }, })), } } export interface RnxCaptureAtlasRegion extends RnxCaptureRegionRequest { /** source rectangle inside the packed atlas, in logical atlas coordinates */ source: RnxCaptureRect } export interface RnxCaptureRegionAtlas { kind: 'png-atlas' producer: 'headless-cpu-app-surface' | 'browser-compositor-app-surface' dataUri: string /** logical atlas dimensions; the encoded PNG uses pixelWidth × pixelHeight */ width: number height: number pixelWidth: number pixelHeight: number regions: RnxCaptureAtlasRegion[] } export interface RnxCaptureFontRegistration { url?: string family: string sourceLabel?: string parsedFamily?: string | null parsedPostScript?: string | null fullName?: string | null } export interface RnxScreenCapture { contractVersion: typeof RNX_SCREEN_CAPTURE_VERSION capturedAt: number tree: RnxCaptureTree /** app fonts registered in the captured tenant, included only when requested */ fonts?: RnxCaptureFontRegistration[] /** optional full-frame diagnostic evidence, never required by tree consumers */ visual?: RnxCaptureVisualEvidence } function captureRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value) } function captureInteger(value: unknown): value is number { return typeof value === 'number' && Number.isSafeInteger(value) } function captureNumber(value: unknown): value is number { return typeof value === 'number' && Number.isFinite(value) } function captureNullableString(value: unknown): value is string | null { return value === null || typeof value === 'string' } function captureRect(value: unknown): value is RnxCaptureRect { return ( captureRecord(value) && captureNumber(value.x) && captureNumber(value.y) && captureNumber(value.width) && captureNumber(value.height) ) } function captureOptionalString(value: unknown): value is string | undefined { return value === undefined || typeof value === 'string' } function captureOptionalNullableString( value: unknown, ): value is string | null | undefined { return value === undefined || captureNullableString(value) } function captureFontRegistration(value: unknown): value is RnxCaptureFontRegistration { return ( captureRecord(value) && typeof value.family === 'string' && value.family.length > 0 && captureOptionalString(value.url) && captureOptionalString(value.sourceLabel) && captureOptionalNullableString(value.parsedFamily) && captureOptionalNullableString(value.parsedPostScript) && captureOptionalNullableString(value.fullName) ) } function capturePoint(value: unknown): value is { x: number; y: number } { return captureRecord(value) && captureNumber(value.x) && captureNumber(value.y) } function captureNode(value: unknown): value is RnxCaptureNode { if ( !captureRecord(value) || !captureInteger(value.nodeId) || !(value.parentNodeId === null || captureInteger(value.parentNodeId)) || !Array.isArray(value.childNodeIds) || !value.childNodeIds.every(captureInteger) || !captureInteger(value.paintIndex) || typeof value.type !== 'string' || !captureNullableString(value.id) || !captureNullableString(value.testID) || !captureNullableString(value.text) || !captureInteger(value.childCount) || !captureRecord(value.accessibility) || !captureRecord(value.interaction) || !captureRecord(value.layout) || !captureRecord(value.style) || !captureRecord(value.source) || !captureRecord(value.props) || !captureRecord(value.inspection) ) { return false } const accessibility = value.accessibility if ( typeof accessibility.accessible !== 'boolean' || !captureNullableString(accessibility.label) || !captureNullableString(accessibility.role) || !captureNullableString(accessibility.hint) || !( accessibility.liveRegion === null || accessibility.liveRegion === 'none' || accessibility.liveRegion === 'polite' || accessibility.liveRegion === 'assertive' ) || !(accessibility.state === null || captureRecord(accessibility.state)) || !(accessibility.value === null || captureRecord(accessibility.value)) ) { return false } const interaction = value.interaction if ( typeof interaction.pressed !== 'boolean' || typeof interaction.pressable !== 'boolean' || typeof interaction.textInput !== 'boolean' || !(interaction.textSelection === null || captureRecord(interaction.textSelection)) || typeof interaction.secureTextEntry !== 'boolean' || !captureNullableString(interaction.placeholder) || !capturePoint(interaction.scrollOffset) || !(interaction.zoomScale === null || captureNumber(interaction.zoomScale)) || typeof interaction.nativePinchEnabled !== 'boolean' || !( interaction.contentOffsetProp === null || captureRecord(interaction.contentOffsetProp) ) || !( interaction.strokeDashoffsetProp === null || captureNumber(interaction.strokeDashoffsetProp) ) ) { return false } const layout = value.layout if ( !captureRect(layout.local) || !(layout.frame === null || captureRect(layout.frame)) || !capturePoint(layout.absolutePosition) || !(layout.visibleFrame === null || captureRect(layout.visibleFrame)) ) { return false } const style = value.style return ( captureRecord(style.resolved) && captureRecord(style.authored) && captureRecord(style.computed) && captureRecord(style.tokenBindings) && (value.textLines === undefined || Array.isArray(value.textLines)) ) } export function isRnxScreenCapture(value: unknown): value is RnxScreenCapture { if ( !captureRecord(value) || value.contractVersion !== RNX_SCREEN_CAPTURE_VERSION || !captureInteger(value.capturedAt) || !captureRecord(value.tree) || !( value.fonts === undefined || (Array.isArray(value.fonts) && value.fonts.length <= 200 && value.fonts.every(captureFontRegistration)) ) || !(value.visual === undefined || captureRecord(value.visual)) ) { return false } const tree = value.tree const visual = value.visual return ( captureRecord(tree.viewport) && captureInteger(tree.viewport.width) && tree.viewport.width > 0 && captureInteger(tree.viewport.height) && tree.viewport.height > 0 && captureInteger(tree.rootNodeId) && (tree.commitRevision === undefined || captureInteger(tree.commitRevision)) && Array.isArray(tree.nodes) && tree.nodes.every(captureNode) && (visual === undefined || (visual.kind === 'png' && (visual.producer === 'headless-cpu-app-surface' || visual.producer === 'browser-compositor-app-surface') && typeof visual.dataUri === 'string' && visual.dataUri.startsWith('data:image/png;base64,') && captureInteger(visual.pixelWidth) && visual.pixelWidth > 0 && captureInteger(visual.pixelHeight) && visual.pixelHeight > 0)) ) } export function isRnxCaptureRegionAtlas(value: unknown): value is RnxCaptureRegionAtlas { if ( !captureRecord(value) || value.kind !== 'png-atlas' || (value.producer !== 'headless-cpu-app-surface' && value.producer !== 'browser-compositor-app-surface') || typeof value.dataUri !== 'string' || !value.dataUri.startsWith('data:image/png;base64,') || !captureNumber(value.width) || value.width <= 0 || !captureNumber(value.height) || value.height <= 0 || !captureInteger(value.pixelWidth) || value.pixelWidth <= 0 || !captureInteger(value.pixelHeight) || value.pixelHeight <= 0 || !Array.isArray(value.regions) || value.regions.length === 0 || value.regions.length > 256 ) { return false } const scale = value.pixelWidth / value.width if ( !Number.isInteger(scale) || scale < 1 || scale > 4 || Math.abs(value.height * scale - value.pixelHeight) > 0.000_001 ) { return false } const keys = new Set() for (const region of value.regions) { if ( !captureRecord(region) || typeof region.key !== 'string' || region.key.length === 0 || keys.has(region.key) || !captureInteger(region.nodeId) || region.nodeId <= 0 || !captureRect(region.frame) || region.frame.width <= 0 || region.frame.height <= 0 || !captureRect(region.source) || region.source.x < 0 || region.source.y < 0 || region.source.width !== region.frame.width || region.source.height !== region.frame.height || region.source.x + region.source.width > value.width || region.source.y + region.source.height > value.height ) { return false } keys.add(region.key) } return true }