import * as THREE from 'three'; import type {Interaction} from '../../../interaction/Interaction'; import {getUIElementKind, isUIElement} from '../../../ui/UIElement'; import {roundContextNumber} from '../../shared/ContextNumberUtils'; import {SemanticIdRegistry} from '../../shared/SemanticIdRegistry'; import { getObjectBounds, hasRenderableDescendant, isSemanticInternalObject, } from '../../shared/SemanticObjectUtils'; import { SemanticBounds, SemanticMetadata, SemanticNode, SemanticSource, SemanticTree, } from '../../shared/SemanticTypes'; type SemanticObject = THREE.Object3D & { disabled?: boolean; text?: string; icon?: string; label?: string; ariaLabel?: string; value?: number; min?: number; max?: number; userData: THREE.Object3D['userData'] & { semantic?: SemanticMetadata; }; }; type SemanticDescription = Pick< SemanticNode, | 'role' | 'name' | 'source' | 'text' | 'traits' | 'disabled' | 'selected' | 'hovered' | 'value' | 'min' | 'max' | 'pointerEvents' | 'interactionEnabled' >; export interface SemanticTreeInternal { tree: SemanticTree; nodeObjects: Map; objectNodeIds: WeakMap; } const tempPosition = new THREE.Vector3(); const tempBoundsCenter = new THREE.Vector3(); const tempBoundsSize = new THREE.Vector3(); const tempBoundsBox = new THREE.Box3(); let snapshotCounter = 0; export function buildSemanticTree({ scene, registry, capturedAt, interaction, }: { scene: THREE.Scene; registry: SemanticIdRegistry; capturedAt: number; interaction?: Interaction; }): SemanticTreeInternal { scene.updateMatrixWorld(true); const nodes: Record = {}; const rootIds: string[] = []; const nodeObjects = new Map(); const objectNodeIds = new WeakMap(); const roundedCapturedAt = roundContextNumber(capturedAt); const snapshotId = `ctx_snapshot_${Math.round(roundedCapturedAt)}_${snapshotCounter++}`; const visit = ( object: THREE.Object3D, semanticParentId: string | undefined ) => { if (object.userData.xrblocksPrivateSelf === true) { for (const child of object.children) { visit(child, semanticParentId); } return; } if (shouldPruneObject(object)) { return; } const semantic = describeSemanticObject(object, interaction); let nextSemanticParentId = semanticParentId; if (semantic) { const id = registry.getNodeId(object); const node = createSemanticNode(object, id, semantic, semanticParentId); nodes[id] = node; nodeObjects.set(id, object); objectNodeIds.set(object, id); if (semanticParentId) { nodes[semanticParentId]?.children.push(id); } else { rootIds.push(id); } nextSemanticParentId = id; } for (const child of object.children) { visit(child, nextSemanticParentId); } }; for (const child of scene.children) { visit(child, undefined); } return { tree: { snapshotId, capturedAt: roundedCapturedAt, rootIds, nodes, }, nodeObjects, objectNodeIds, }; } function shouldPruneObject(object: THREE.Object3D): boolean { const maybeSemantic = (object as SemanticObject).userData.semantic; if (maybeSemantic?.hidden) { return true; } return isSemanticInternalObject(object); } function describeSemanticObject( object: THREE.Object3D, interaction?: Interaction ): SemanticDescription | null { const semanticObject = object as SemanticObject; const override = semanticObject.userData.semantic; const hasExplicitIdentity = hasExplicitSemanticIdentity(object); const role = resolveRole(object); if (!role) { return null; } if (!hasExplicitIdentity) { const isImplementationMesh = object instanceof THREE.Mesh && hasSemanticAncestor(object); if (isImplementationMesh || isLayoutOnlyContainer(object, role)) { return null; } } const disabled = override?.disabled ?? inferDisabled(object); return { role, name: override?.name ?? inferName(object), source: override?.source ?? inferSource(object), text: override?.text ?? inferText(object), traits: mergeTraits(inferTraits(object, disabled), override?.traits), disabled, selected: interaction?.isSelectingAt(object), hovered: interaction?.isHovered(object), pointerEvents: object.xb?.pointerEvents ?? 'auto', interactionEnabled: object.xb?.interactionEnabled ?? true, ...inferValue(object), }; } function hasSemanticAncestor(object: THREE.Object3D): boolean { let parent = object.parent; while (parent) { const hasExplicitIdentity = hasExplicitSemanticIdentity(parent); const role = resolveRole(parent); if (role && (hasExplicitIdentity || !isLayoutOnlyContainer(parent, role))) { return true; } parent = parent.parent; } return false; } function hasExplicitSemanticIdentity(object: THREE.Object3D): boolean { const semantic = (object as SemanticObject).userData.semantic; return Boolean(semantic?.role || semantic?.name); } function resolveRole(object: THREE.Object3D): string { const semantic = (object as SemanticObject).userData.semantic; if (semantic?.role) return semantic.role; const inferredRole = inferRole(object); if (inferredRole) return inferredRole; return semantic?.name ? inferExplicitRole(object) : ''; } function inferRole(object: THREE.Object3D): string { if (isUIElement(object)) { const kind = getUIElementKind(object); if (kind === 'button') return 'button'; if (kind === 'slider') return 'slider'; if (kind === 'text') return 'text'; if (kind === 'image' || kind === 'icon') return 'image'; return 'group'; } if (object instanceof THREE.Mesh) return 'object'; if (object instanceof THREE.Group && hasRenderableDescendant(object)) { return object.name ? 'group' : ''; } return ''; } function inferExplicitRole(object: THREE.Object3D): string { return object.children.length > 0 ? 'group' : 'object'; } function inferName(object: THREE.Object3D): string { const semanticObject = object as SemanticObject; return ( semanticObject.ariaLabel ?? semanticObject.label ?? semanticObject.text ?? semanticObject.icon ?? object.name ?? `${object.type}_${object.id}` ); } function inferText(object: THREE.Object3D): string | undefined { return (object as SemanticObject).text; } function inferSource(object: THREE.Object3D): SemanticSource { if (isUIElement(object)) return 'xrblocks'; return 'three'; } function inferTraits( object: THREE.Object3D, disabled: boolean | undefined ): string[] | undefined { const traits = new Set(); if (object.xb?.manipulation) traits.add('manipulable'); if ( isUIElement(object) && (getUIElementKind(object) === 'button' || getUIElementKind(object) === 'slider') && object.xb?.interactionEnabled !== false && !disabled ) { traits.add('selectable'); } return traits.size ? [...traits] : undefined; } function mergeTraits( inferred: string[] | undefined, explicit: string[] | undefined ): string[] | undefined { if (!inferred?.length && !explicit?.length) return undefined; return [...new Set([...(inferred ?? []), ...(explicit ?? [])])]; } function inferValue( object: THREE.Object3D ): Pick { if (!isUIElement(object) || getUIElementKind(object) !== 'slider') return {}; const slider = object as SemanticObject; return {value: slider.value, min: slider.min, max: slider.max}; } function inferDisabled(object: THREE.Object3D): boolean | undefined { return (object as SemanticObject).disabled; } function isLayoutOnlyContainer(object: THREE.Object3D, role: string): boolean { const className = object.constructor.name; if (role !== 'group') { return false; } return !object.name && (className === 'Object3D' || className === 'Group'); } function createSemanticNode( object: THREE.Object3D, id: string, semantic: NonNullable>, parentId: string | undefined ): SemanticNode { object.updateMatrixWorld(true); object.getWorldPosition(tempPosition); const node: SemanticNode = { id, role: semantic.role, name: semantic.name, visible: isEffectivelyVisible(object), pointerEvents: semantic.pointerEvents, interactionEnabled: semantic.interactionEnabled, position: [ roundContextNumber(tempPosition.x), roundContextNumber(tempPosition.y), roundContextNumber(tempPosition.z), ], children: [], objectId: object.id, source: semantic.source, type: object.constructor.name || object.type, }; if (parentId) node.parentId = parentId; if (semantic.text) node.text = semantic.text; if (semantic.traits?.length) node.traits = semantic.traits; if (semantic.disabled !== undefined) node.disabled = semantic.disabled; if (semantic.selected !== undefined) node.selected = semantic.selected; if (semantic.hovered !== undefined) node.hovered = semantic.hovered; if (semantic.value !== undefined) node.value = semantic.value; if (semantic.min !== undefined) node.min = semantic.min; if (semantic.max !== undefined) node.max = semantic.max; const bounds = getSemanticBounds(object); if (bounds) node.bounds = bounds; return node; } function isEffectivelyVisible(object: THREE.Object3D): boolean { let current: THREE.Object3D | null = object; while (current) { if (!current.visible) return false; current = current.parent; } return true; } function getSemanticBounds(object: THREE.Object3D): SemanticBounds | undefined { const bounds = getObjectBounds(object, tempBoundsBox); if (!bounds) { return undefined; } const center = bounds.getCenter(tempBoundsCenter); const size = bounds.getSize(tempBoundsSize); return { center: [ roundContextNumber(center.x), roundContextNumber(center.y), roundContextNumber(center.z), ], size: [ roundContextNumber(size.x), roundContextNumber(size.y), roundContextNumber(size.z), ], }; }