import * as THREE from 'three'; import {AI} from '../ai/AI'; import {AIOptions} from '../ai/AIOptions'; import {markDebugFailed, markDebugReady} from '../debug/DebugGlobals'; import {XRDeviceCamera} from '../camera/XRDeviceCamera'; import {Context} from '../context/Context'; import {ContextOptions} from '../context/ContextOptions'; import {SceneOptions} from '../context/scene/SceneOptions'; import {UI_OVERLAY_LAYER} from '../constants'; import {Depth} from '../depth/Depth'; import {DepthOptions} from '../depth/DepthOptions'; import {Hands} from '../input/Hands'; import {GestureRecognition} from '../input/gestures/GestureRecognition'; import {GestureRecognitionOptions} from '../input/gestures/GestureRecognitionOptions.js'; import {HeadGestureRecognitionOptions} from '../input/headGestures/HeadGestureRecognitionOptions.js'; import type {PoseEstimator} from '../input/gestures/GestureTypes'; import {StrokeRecognitionOptions} from '../input/strokes/StrokeRecognitionOptions'; import {Input} from '../input/Input'; import {Lighting} from '../lighting/Lighting'; import {Physics} from '../physics/Physics'; import {Simulator} from '../simulator/Simulator'; import {SimulatorOptions} from '../simulator/SimulatorOptions'; import {CoreSound} from '../sound/CoreSound'; import {SoundOptions} from '../sound/SoundOptions'; import {UI} from '../ui/UI'; import {callInitWithDependencyInjection} from '../utils/DependencyInjection'; import {loadingSpinnerManager} from '../utils/LoadingSpinnerManager'; import {traverseUtil} from '../utils/SceneGraphUtils'; import {DragManager} from '../ux/DragManager'; import {World} from '../world/World'; import {WorldOptions} from '../world/WorldOptions'; import {MeshDetectionOptions} from '../world/mesh/MeshDetectionOptions'; import {Registry} from './components/Registry'; import {ScreenshotSynthesizer} from './components/ScreenshotSynthesizer'; import {SimulationTimer} from './components/SimulationTimer'; import {ScriptsManager} from './components/ScriptsManager'; import {WaitFrame} from './components/WaitFrame'; import { WebXRSessionEventType, WebXRSessionManager, } from './components/WebXRSessionManager'; import {XRButton} from './components/XRButton'; import {XREffects} from './components/XREffects'; import {XRTransition} from './components/XRTransition'; import {Options} from './Options'; import {UIKitOptions} from './UIKitOptions'; import {Script} from './Script'; import {User} from './User'; import {PermissionsManager} from './components/PermissionsManager'; import {XRSystems} from './components/XRSystems'; /** * Core is the central engine of the XR Blocks framework, acting as a * singleton manager for all XR subsystems. Its primary goal is to abstract * low-level WebXR and THREE.js details, providing a simplified and powerful API * for developers and AI agents to build interactive XR applications. */ export class Core { static instance?: Core; /** * Component responsible for capturing screenshots of the XR scene for AI. */ screenshotSynthesizer = new ScreenshotSynthesizer(); /** * Component responsible for waiting for the next frame. */ waitFrame = new WaitFrame(); /** * Registry used for dependency injection on existing subsystems. */ registry = new Registry(); /** * A timer for tracking time deltas. Call timer.getDelta() or getDeltaTime(). */ timer = new THREE.Timer(); private simulationTimer = new SimulationTimer(); /** Manages hand, mouse, gaze inputs. */ input = new Input(); /** The main camera for rendering. */ camera = new THREE.PerspectiveCamera(); /** The root scene graph for all objects. */ scene = new THREE.Scene(); /** Represents the user in the XR scene. */ user = new User(); /** Manages all UI elements. */ ui = new UI(); /** Manages all (spatial) audio playback. */ sound = new CoreSound(); /** A container to hold all the systems in the scene hierarchy. */ xrSystemsGroup = new XRSystems(); private renderSceneCallback = (cameraOverride?: THREE.Camera) => this.renderScene(cameraOverride); /** Manages the desktop XR simulator. */ simulator = new Simulator(this.renderSceneCallback); /** Manages drag-and-drop interactions. */ dragManager = new DragManager(); /** Manages real-world understanding: planes, meshes, objects, and sounds. */ world = new World(); /** Manages agent-facing observations of the app/session. */ context = new Context(); /** A shared texture loader. */ textureLoader = new THREE.TextureLoader(); private webXRSettings: XRSessionInit = {}; /** Whether the XR simulator is currently active. */ simulatorRunning = false; private startingSimulator?: Promise; private _isPaused = false; private isSteppingFrame = false; private manualStepTime = 0; private _renderer?: THREE.WebGLRenderer; options!: Options; deviceCamera?: XRDeviceCamera; depth = new Depth(); lighting?: Lighting; physics?: Physics; xrButton?: XRButton; effects?: XREffects; ai = new AI(); poseEstimation?: PoseEstimator; gestureRecognition?: GestureRecognition; transition?: XRTransition; currentFrame?: XRFrame; scriptsManager = new ScriptsManager(async (script: Script) => { await callInitWithDependencyInjection(script, this.registry, this); if (this.physics) { await script.initPhysics(this.physics); } }); renderSceneOverride?: ( renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera ) => void; webXRSessionManager?: WebXRSessionManager; permissionsManager = new PermissionsManager(); /** * The WebGL renderer, created during {@link Core.init}. Reading it before * `init()` has run returns `undefined` and logs a one-time warning. */ get renderer(): THREE.WebGLRenderer { if (!this._renderer) { console.warn( 'xb.core.renderer is not available until xb.init() creates it. ' + "Access it in or after your Script's init() method." ); } return this._renderer!; } set renderer(renderer: THREE.WebGLRenderer) { this._renderer = renderer; } get isPaused() { return this._isPaused; } pause() { this._isPaused = true; this.simulationTimer.pause(); } resume() { this._isPaused = false; } stepFrame(dtMs = 16.67) { if (this.isSteppingFrame) { throw new Error( 'Core.stepFrame() cannot be called while already stepping.' ); } this.isSteppingFrame = true; try { this.simulationTimer.step(dtMs, this.timer.getTimescale()); this.manualStepTime += dtMs; this.update(this.manualStepTime, undefined as unknown as XRFrame); if (this.physics) { this.physicsStep(); } } finally { this.isSteppingFrame = false; } } /** * Core is a singleton manager that manages all XR "blocks". * It initializes core components and abstractions like the scene, camera, * user, UI, AI, and input managers. */ constructor() { if (Core.instance) { return Core.instance; } Core.instance = this; this.scene.name = 'XR Blocks Scene'; this.scene.add(this.xrSystemsGroup); this.xrSystemsGroup.add( this.user, this.dragManager, this.ui, this.sound, this.world, this.context ); this.registry.register(this.registry); this.registry.register(this); this.registry.register(this.waitFrame); this.registry.register(this.screenshotSynthesizer); this.registry.register(this.simulationTimer); this.registry.register(this.scene); this.registry.register(this.timer); this.registry.register(this.input); this.registry.register(this.user); this.registry.register(this.ui); this.registry.register(this.sound); this.registry.register(this.dragManager); this.registry.register(this.simulator); this.registry.register(this.simulator.navMesh); this.registry.register(this.scriptsManager); this.registry.register(this.depth); this.registry.register(this.world); this.registry.register(this.context); this.registry.register(this.xrSystemsGroup); } /** * Initializes the Core system with a given set of options. This includes * setting up the renderer, enabling features like controllers, depth * sensing, and physics, and starting the render loop. * @param options - Configuration options for the * session. */ async init(options = new Options()) { try { await this.initialize(options); markDebugReady(this); } catch (error) { markDebugFailed(this, error); throw error; } } private async initialize(options: Options) { loadingSpinnerManager.showSpinner(); this.registry.register(options, Options); this.registry.register(options.depth, DepthOptions); this.registry.register(options.simulator, SimulatorOptions); this.registry.register(options.world, WorldOptions); this.registry.register(options.context, ContextOptions); this.registry.register(options.context.scene, SceneOptions); this.registry.register(options.world.meshes, MeshDetectionOptions); this.registry.register(options.uikit, UIKitOptions); this.registry.register(options.ai, AIOptions); this.registry.register(options.sound, SoundOptions); this.registry.register(options.gestures, GestureRecognitionOptions); this.registry.register(options.headGestures, HeadGestureRecognitionOptions); this.registry.register(options.strokes, StrokeRecognitionOptions); if (options.transition.enabled) { this.transition = new XRTransition(); this.user.add(this.transition); this.registry.register(this.transition); } this.camera.copy( new THREE.PerspectiveCamera( /*fov=*/ 90, window.innerWidth / window.innerHeight, /*near=*/ options.camera.near, /*far=*/ options.camera.far ) ); this.registry.register(this.camera, THREE.Camera); this.registry.register(this.camera, THREE.PerspectiveCamera); this.renderer = new THREE.WebGLRenderer({ canvas: options.canvas, antialias: options.antialias, stencil: options.stencil, alpha: true, logarithmicDepthBuffer: options.logarithmicDepthBuffer, }); this.renderer.setPixelRatio(window.devicePixelRatio); this.renderer.setSize(window.innerWidth, window.innerHeight); this.renderer.xr.enabled = true; // disable built-in occlusion this.renderer.xr.getDepthSensingMesh = function () { return null; }; this.registry.register(this.renderer); if (options.uikit.enabled) { this.renderer.localClippingEnabled = true; if (options.uikit.reversePainterSortStable) { this.renderer.setTransparentSort( options.uikit.reversePainterSortStable ); } } this.renderer.xr.setReferenceSpaceType(options.referenceSpaceType); // For desktop simulator: window.addEventListener('resize', this.onWindowResize); if (!options.canvas) { const xrContainer = document.createElement('div'); document.body.appendChild(xrContainer); xrContainer.appendChild(this.renderer.domElement); } this.options = options; this.scriptsManager.catchExceptions = options.catchScriptExceptions; // Sets up input. Head gestures are camera-only and do not require controllers. this.input.init({ scene: this.scene, systemsGroup: this.xrSystemsGroup, options: options, renderer: this.renderer, }); if (options.controllers.enabled) { this.input.bindSelectStart(this.scriptsManager.callSelectStart); this.input.bindSelectEnd(this.scriptsManager.callSelectEnd); this.input.bindSelect(this.scriptsManager.callSelect); this.input.bindSqueezeStart(this.scriptsManager.callSqueezeStart); this.input.bindSqueezeEnd(this.scriptsManager.callSqueezeEnd); this.input.bindSqueeze(this.scriptsManager.callSqueeze); this.input.bindKeyDown(this.scriptsManager.callKeyDown); this.input.bindKeyUp(this.scriptsManager.callKeyUp); } // Sets up device camera. if (options.deviceCamera?.enabled) { this.deviceCamera = new XRDeviceCamera(options.deviceCamera); this.deviceCamera.setRenderer(this.renderer); this.registry.register(this.deviceCamera); this.context.setDeviceCamera(this.deviceCamera); } const webXRRequiredFeatures: string[] = options.webxrRequiredFeatures; const webXROptionalFeatures: string[] = options.webxrOptionalFeatures; // Use camera-access when the browser supports it. if (options.deviceCamera?.enabled) { webXROptionalFeatures.push('camera-access'); } this.webXRSettings.requiredFeatures = webXRRequiredFeatures; this.webXRSettings.optionalFeatures = webXROptionalFeatures; // Sets up depth. if (options.depth.enabled) { webXRRequiredFeatures.push('depth-sensing'); webXRRequiredFeatures.push('local-floor'); this.webXRSettings.depthSensing = { usagePreference: options.depth.usagePreference, dataFormatPreference: options.depth.dataFormatPreference, depthTypeRequest: options.depth.depthTypeRequest, matchDepthView: options.depth.matchDepthView, }; this.depth.init( this.camera, options.depth, this.renderer, this.registry, this.scene ); } if (options.hands.enabled) { webXRRequiredFeatures.push('hand-tracking'); this.user.hands = new Hands(this.input.hands); if (options.gestures.enabled) { this.poseEstimation = options.gestures.poseEstimator; this.gestureRecognition = new GestureRecognition(); this.xrSystemsGroup.add(this.gestureRecognition); this.registry.register(this.poseEstimation); this.registry.register(this.gestureRecognition); } } // World-sensing and lighting features are requested as optional so that // browsers lacking one of them still enter the immersive session instead // of rejecting it outright; the subsystems no-op when the data is absent. if (options.world.planes.enabled) { webXROptionalFeatures.push('plane-detection'); } if (options.world.meshes.enabled) { webXROptionalFeatures.push('mesh-detection'); } // Sets up lighting. if (options.lighting.enabled) { webXROptionalFeatures.push('light-estimation'); this.lighting = new Lighting(); this.lighting.init( options.lighting, this.renderer, this.scene, this.depth ); } // Sets up physics. if (options.physics && options.physics.RAPIER) { this.physics = new Physics(); this.registry.register(this.physics); await this.physics.init({physicsOptions: options.physics}); if (options.depth.enabled) { this.depth.depthMesh?.initRapierPhysics( this.physics.RAPIER, this.physics.blendedWorld ); } } this.webXRSessionManager = new WebXRSessionManager( this.renderer, this.webXRSettings, options.xrSessionMode ); this.webXRSessionManager.addEventListener( WebXRSessionEventType.SESSION_START, (event) => this.onXRSessionStarted(event.session) ); this.webXRSessionManager.addEventListener( WebXRSessionEventType.SESSION_END, this.onXRSessionEnded ); // Sets up xrButton. let shouldAutostartSimulator = this.options.xrButton.alwaysAutostartSimulator; if (!shouldAutostartSimulator && options.xrButton.enabled) { this.xrButton = new XRButton( this.webXRSessionManager, this.permissionsManager, options.xrButton?.appTitle, options.xrButton?.appDescription, options.xrButton?.startText, options.xrButton?.endText, options.xrButton?.invalidText, options.xrButton?.startSimulatorText, options.xrButton?.showEnterSimulatorButton, this.startSimulator, options.permissions ); document.body.appendChild(this.xrButton.domElement); } this.webXRSessionManager.addEventListener( WebXRSessionEventType.UNSUPPORTED, () => { if (this.options.enableSimulator) { this.xrButton?.domElement.remove(); shouldAutostartSimulator = true; } } ); await this.webXRSessionManager.initialize(); // Sets up postprocessing effects. if (options.usePostprocessing) { this.effects = new XREffects(this.renderer, this.scene, this.timer); this.simulator.effects = this.effects; } // Sets up AI services. if (options.ai.enabled) { this.registry.register(this.ai); this.xrSystemsGroup.add(this.ai); // Manually init the script in case other scripts rely on it. await this.scriptsManager.initScript(this.ai); } await this.scriptsManager.syncScriptsWithScene(this.scene); this.renderer.setAnimationLoop(this.update); if (this.physics) { setInterval(this.physicsStep, 1000 * this.physics.timestep); } if (this.options.reticles.enabled) { this.input.addReticles(); } if (shouldAutostartSimulator) { await this.startSimulator(); } if (!loadingSpinnerManager.isLoading) { loadingSpinnerManager.hideSpinner(); } } /** * The main update loop, called every frame by the renderer. It orchestrates * all per-frame updates for subsystems and scripts. * * Order: * 1. Depth * 2. World Perception * 3. Input / Reticles / UIs * 4. Scripts * @param time - The current time in milliseconds. * @param frame - The WebXR frame object, if in an XR session. */ private update = (time: number, frame: XRFrame) => { if (this._isPaused && !this.isSteppingFrame) { return; } this.currentFrame = frame; this.manualStepTime = Math.max(this.manualStepTime, time); if (!this.isSteppingFrame) { this.simulationTimer.update(time, this.timer.getTimescale()); } this.timer.update(time); if (this.simulatorRunning) { this.simulator.simulatorUpdate(); } this.depth.update(frame); // Update XR camera fallback textures. if (this.deviceCamera?.isUsingXRCameraAccess) { this.deviceCamera.updateXRCamera(frame); } if (this.lighting) { this.lighting.update(); } // Traverse the scene to find all scripts. this.scriptsManager.syncScriptsWithScene(this.scene); // Force matrix updates since there is no active renderer render loop in headless/test environments. this.scene.updateMatrixWorld(true); // Updates reticles and UIs. this.scriptsManager.resetUX(); this.input.update(); // Updates scripts with user interactions. for (const controller of this.input.controllers) { if (controller.userData.selected) { this.scriptsManager.callSelecting(controller); } } for (const controller of this.input.controllers) { if (controller.userData.squeezing) { this.scriptsManager.callSqueezing(controller); } } // Run callbacks that use wait frame. this.waitFrame.onFrame(); // Updates renderings. this.scriptsManager.update(time, frame); this.renderSimulatorAndScene(); this.screenshotSynthesizer.onAfterRender( this.renderer, this.renderSceneCallback, this.deviceCamera ); if (this.simulatorRunning) { this.simulator.renderSimulatorScene(); } }; /** * Advances the physics simulation by a fixed timestep and calls the * corresponding physics update on all active scripts. */ private physicsStep = () => { if (this._isPaused && !this.isSteppingFrame) { return; } this.physics!.physicsStep(); this.scriptsManager.physicsStep(); }; /** * Lifecycle callback executed when an XR session starts. Notifies all active * scripts. * @param session - The newly started WebXR session. */ private async onXRSessionStarted(session: XRSession) { if (this.options.deviceCamera?.enabled) { await this.deviceCamera!.init(); } this.scriptsManager.onXRSessionStarted(session); } private startSimulator = async () => { if (this.simulatorRunning) return; if (this.startingSimulator) return this.startingSimulator; this.startingSimulator = (async () => { this.xrButton?.domElement.remove(); this.xrSystemsGroup.add(this.simulator); await this.scriptsManager.initScript(this.simulator); this.onSimulatorStarted(); })(); try { await this.startingSimulator; } finally { this.startingSimulator = undefined; } }; /** * Lifecycle callback executed when an XR session ends. Notifies all active * scripts. */ private onXRSessionEnded = () => { this.scriptsManager.onXRSessionEnded(); }; /** * Lifecycle callback executed when the desktop simulator starts. Notifies * all active scripts. */ private onSimulatorStarted() { this.simulatorRunning = true; this.scriptsManager.onSimulatorStarted(); if (this.lighting) { this.lighting.simulatorRunning = true; } } /** * Handles browser window resize events to keep the camera and renderer * synchronized. */ private onWindowResize = () => { this.camera.aspect = window.innerWidth / window.innerHeight; this.camera.updateProjectionMatrix(); this.renderer.setSize(window.innerWidth, window.innerHeight); }; private renderSimulatorAndScene() { if (this.simulatorRunning) { this.simulator.renderScene(); } else { this.renderScene(); } } private renderScene(cameraOverride?: THREE.Camera) { if (this.renderSceneOverride) { this.renderSceneOverride( this.renderer, this.scene, cameraOverride ?? this.camera ); } else if (this.effects) { this.effects.render(); } else { this.renderer.render(this.scene, cameraOverride ?? this.camera); if ( traverseUtil(this.scene, (node: THREE.Object3D) => node.layers.isEnabled(UI_OVERLAY_LAYER) ) ) { const originalLayers = this.camera.layers.mask; this.camera.layers.set(UI_OVERLAY_LAYER); this.renderer.render(this.scene, this.camera); this.camera.layers.mask = originalLayers; } } } }