/** * Copyright 2025 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * @file xrblocks.js * @version v0.12.0 * @commitid 6ae2b5e * @builddate 2026-04-01T20:37:04.784Z * @description XR Blocks SDK, built from source with the above commit ID. * @agent When using with Gemini to create XR apps, use **Gemini Canvas** mode, * and follow rules below: * 1. Include the following importmap for maximum compatibility: "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js", "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/", "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js", "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js", "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js", "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022", "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs", "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js", "lit/": "https://esm.run/lit@3/", 2. If the app focus on standalone objects, spawn it in front of the user in WebXR and rescale to reasonable physical size. Wrap them with xb.ModelViewer and make sure users can drag the platform to move it around in XR. 3. When rendering a large scene, remember users may navigate the scene in a physical world space, also add locomotion methods like pinch to teleport. 4. Do not halluciate mode files --- use either public high-quality assets, or generate from primitive shapes of use vox formats for voxels or lego-styles. */ import * as GoogleGenAITypes from '@google/genai'; import * as THREE from 'three'; import RAPIER_NS from 'rapier3d'; import OpenAIType from 'openai'; import { Pass, FullScreenQuad } from 'three/addons/postprocessing/Pass.js'; import { TemplateResult } from 'lit'; import { GLTFLoader, GLTF } from 'three/addons/loaders/GLTFLoader.js'; import TroikaThreeText from 'troika-three-text'; import * as _sparkjsdev_spark from '@sparkjsdev/spark'; import { SparkRenderer } from '@sparkjsdev/spark'; /** * A 3D visual marker used to indicate a user's aim or interaction * point in an XR scene. It orients itself to surfaces it intersects with and * provides visual feedback for states like "pressed". */ declare class Reticle extends THREE.Mesh { /** Text description of the PanelMesh */ name: string; editorIcon: string; /** Prevents the reticle itself from being a target for raycasting. */ ignoreReticleRaycast: boolean; /** The world-space direction vector of the ray that hit the target. */ direction: THREE.Vector3; /** Ensures the reticle is drawn on top of other transparent objects. */ renderOrder: number; /** The smoothing factor for rotational slerp interpolation. */ rotationSmoothing: number; /** The z-offset to prevent visual artifacts (z-fighting). */ offset: number; /** The most recent intersection data that positioned this reticle. */ intersection?: THREE.Intersection; /** Object on which the reticle is hovering. */ targetObject?: THREE.Object3D; private readonly originalNormal; private readonly newRotation; private readonly objectRotation; private readonly normalVector; /** * Creates an instance of Reticle. * @param rotationSmoothing - A factor between 0.0 (no smoothing) and * 1.0 (no movement) to smoothly animate orientation changes. * @param offset - A small z-axis offset to prevent z-fighting. * @param size - The radius of the reticle's circle geometry. * @param depthTest - Determines if the reticle should be occluded by other * objects. Defaults to `false` to ensure it is always visible. */ constructor(rotationSmoothing?: number, offset?: number, size?: number, depthTest?: boolean); /** * Orients the reticle to be flush with a surface, based on the surface * normal. It smoothly interpolates the rotation for a polished visual effect. * @param normal - The world-space normal of the surface. */ setRotationFromNormalVector(normal: THREE.Vector3): void; /** * Updates the reticle's complete pose (position and rotation) from a * raycaster intersection object. * @param intersection - The intersection data from a raycast. */ setPoseFromIntersection(intersection: THREE.Intersection): void; /** * Sets the color of the reticle via its shader uniform. * @param color - The color to apply. */ setColor(color: THREE.Color | number | string): void; /** * Gets the current color of the reticle. * @returns The current color from the shader uniform. */ getColor(): THREE.Color; /** * Sets the visual state of the reticle to "pressed" or "unpressed". * This provides visual feedback to the user during interaction. * @param pressed - True to show the pressed state, false otherwise. */ setPressed(pressed: boolean): void; /** * Sets the pressed state as a continuous value for smooth animations. * @param pressedAmount - A value from 0.0 (unpressed) to 1.0 (fully * pressed). */ setPressedAmount(pressedAmount: number): void; /** * Overrides the default raycast method to make the reticle ignored by * raycasters. */ raycast(): void; } interface ControllerEventMap extends THREE.Object3DEventMap { connected: { target: Controller; data?: XRInputSource; }; disconnected: { target: Controller; data?: XRInputSource; }; select: { target: Controller; data?: XRInputSource; }; selectstart: { target: Controller; data?: XRInputSource; }; selectend: { target: Controller; data?: XRInputSource; }; squeeze: { target: Controller; data?: XRInputSource; }; squeezestart: { target: Controller; data?: XRInputSource; }; squeezeend: { target: Controller; data?: XRInputSource; }; } interface Controller extends THREE.Object3D { reticle?: Reticle; gamepad?: Gamepad; inputSource?: Partial; } interface ControllerEvent { type: keyof ControllerEventMap; target: Controller; data?: Partial; } type RAPIERCompat = typeof RAPIER_NS & { init?: () => Promise; }; declare class PhysicsOptions { /** * The target frames per second for the physics simulation loop. */ fps: number; /** * The global gravity vector applied to the physics world. */ gravity: { x: number; y: number; z: number; }; /** * If true, the `Physics` manager will automatically call `world.step()` * on its fixed interval. Set to false if you want to control the * simulation step manually. */ worldStep: boolean; /** * If true, an event queue will be created and passed to `world.step()`, * enabling the handling of collision and contact events. */ useEventQueue: boolean; /** * Instance of RAPIER. */ RAPIER?: RAPIERCompat; } /** * Integrates the RAPIER physics engine into the XRCore lifecycle. * It sets up the physics in a blended world that combines virtual and physical * objects, steps the simulation forward in sync with the application's * framerate, and manages the lifecycle of physics-related objects. */ declare class Physics { initialized: boolean; options?: PhysicsOptions; RAPIER: RAPIERCompat; fps: number; blendedWorld: RAPIER_NS.World; eventQueue: RAPIER_NS.EventQueue; get timestep(): number; /** * Asynchronously initializes the RAPIER physics engine and creates the * blendedWorld. This is called in Core before the physics simulation starts. */ init({ physicsOptions }: { physicsOptions: PhysicsOptions; }): Promise; /** * Advances the physics simulation by one step. */ physicsStep(): void; /** * Frees the memory allocated by the RAPIER physics blendedWorld and event * queue. This is crucial for preventing memory leaks when the XR session * ends. */ dispose(): void; } /** * Misc collection of types not specific to any XR Blocks module. */ type Constructor = new (...args: any[]) => T; type ShaderUniforms = { [uniform: string]: THREE.IUniform; }; /** * Defines the structure for a shader object compatible with PanelMesh, * requiring uniforms, a vertex shader, and a fragment shader. */ interface Shader { uniforms: ShaderUniforms; vertexShader: string; fragmentShader: string; defines?: { [key: string]: unknown; }; } /** * A recursive readonly type. */ type DeepReadonly = T extends (...args: any[]) => any ? T : T extends object ? { readonly [P in keyof T]: DeepReadonly; } : T; /** * A recursive partial type. */ type DeepPartial = T extends (...args: any[]) => any ? T : T extends object ? { [P in keyof T]?: DeepPartial; } : T; /** * UX manages the user experience (UX) state for an interactive object in * the scene. It tracks interaction states like hover, * selection, and dragging for multiple controllers. */ declare class UX { /** * The object this UX state manager is attached to. */ parent: THREE.Object3D; /** * Indicates if the parent object can be dragged. */ draggable: boolean; /** * Indicates if the parent object can be selected. */ selectable: boolean; /** * Indicates if the parent object can be touched. */ touchable: boolean; /** * An array tracking the selection state for each controller. * `selected[i]` is true if controller `i` is selecting the object. */ selected: boolean[]; /** * An array tracking the hover state for each controller. * `hovered[i]` is true if controller `i` is hovering over the object. */ hovered: boolean[]; /** * An array tracking the touch state for each controller. * `touched[i]` is true if controller `i` is touching over the object. */ touched: boolean[]; /** * An array tracking the drag state for each controller. */ activeDragged: boolean[]; /** * An array storing the 3D position of the last intersection for each * controller. */ positions: THREE.Vector3[]; /** * An array storing the distance of the last intersection for each controller. */ distances: number[]; /** * An array storing the UV coordinates of the last intersection for each * controller. */ uvs: THREE.Vector2[]; /** * The initial position of the object when a drag operation begins. */ initialPosition: THREE.Vector3; /** * The initial distance from the controller to the object at the start of a * drag for computing the relative dragging distances and angles. */ initialDistance?: number; /** * @param parent - The script or object that owns this UX instance. */ constructor(parent: THREE.Object3D); /** * Checks if the object is currently being hovered by any controller. */ isHovered(): boolean; /** * Checks if the object is currently being selected by any controller. */ isSelected(): boolean; /** * Checks if the object is currently being dragged by any controller. */ isDragging(): boolean; /** * Updates the interaction state for a specific controller based on a new * intersection. This is internally called by the core input system when a * raycast hits the parent object. * @param controller - The controller performing the * interaction. * @param intersection - The raycast intersection data. */ update(controller: THREE.Object3D, intersection: THREE.Intersection): void; /** * Ensures that the internal arrays for tracking states are large enough to * accommodate a given controller ID. * @param id - The controller ID to ensure exists. */ initializeVariablesForId(id: number): void; /** * Checks if the intersection object belongs to this UX's attached Script. * Allow overriding this function for more complex objects with multiple * meshes. * @param intersection - The raycast intersection to check. * @returns True if the intersection is relevant to this UX's parent object. */ isRelevantIntersection(intersection: THREE.Intersection): boolean; /** * Resets the hover and selection states for all controllers. This is * typically called at the beginning of each frame. */ reset(): void; /** * Gets the IDs of up to two controllers that are currently hovering over the * parent object, always returning a two-element array. This is useful for * shaders or components like Panels that expect a fixed number of interaction * points. * * @returns A fixed-size two-element array. Each element is either a * controller ID (e.g., 0, 1) or null. */ getPrimaryTwoControllerIds(): number[]; } interface SelectEvent { target: Controller; } interface ObjectTouchEvent { handIndex: number; touchPosition: THREE.Vector3; } interface ObjectGrabEvent { handIndex: number; hand: THREE.Object3D; } interface KeyEvent { code: string; } /** * The Script class facilities development by providing useful life cycle * functions similar to MonoBehaviors in Unity. * * Each Script object is an independent THREE.Object3D entity within the * scene graph. * * See /docs/manual/Scripts.md for the full documentation. * * It manages user, objects, and interaction between user and objects. * See `/templates/0_basic/` for an example to start with. * * * If the class does not extends View, it can still bind the above three * function, where the engine ignores whether reticle exists. * * # Supported (native WebXR) functions to extend: * * onSelectStart(event) * onSelectEnd(event) * */ declare function ScriptMixin>(base: TBase): { new (...args: any[]): { ux: UX; isXRScript: boolean; /** * Initializes an instance with XR controllers, grips, hands, raycaster, and * default options. We allow all scripts to quickly access its user (e.g., * user.isSelecting(), user.hands), world (e.g., physical depth mesh, * lighting estimation, and recognized objects), and scene (the root of * three.js's scene graph). If this returns a promise, we will wait for it. */ init(_?: object): void | Promise; /** * Runs per frame. */ update(_time?: number, _frame?: XRFrame): void; /** * Enables depth-aware interactions with physics. See /demos/ballpit */ initPhysics(_physics: Physics): void | Promise; physicsStep(): void; onXRSessionStarted(_session?: XRSession): void; onXRSessionEnded(): void; onSimulatorStarted(): void; /** * Called whenever pinch / mouse click starts, globally. * @param _event - event.target holds its controller */ onSelectStart(_event: SelectEvent): void; /** * Called whenever pinch / mouse click discontinues, globally. * @param _event - event.target holds its controller */ onSelectEnd(_event: SelectEvent): void; /** * Called whenever pinch / mouse click successfully completes, globally. * @param _event - event.target holds its controller. */ onSelect(_event: SelectEvent): void; /** * Called whenever pinch / mouse click is happening, globally. */ onSelecting(_event: SelectEvent): void; /** * Called on keyboard keypress. * @param _event - Event containing `.code` to read the keyboard key. */ onKeyDown(_event: KeyEvent): void; onKeyUp(_event: KeyEvent): void; /** * Called whenever gamepad trigger starts, globally. * @param _event - event.target holds its controller. */ onSqueezeStart(_event: SelectEvent): void; /** * Called whenever gamepad trigger stops, globally. * @param _event - event.target holds its controller. */ onSqueezeEnd(_event: SelectEvent): void; /** * Called whenever gamepad is being triggered, globally. */ onSqueezing(_event: SelectEvent): void; /** * Called whenever gamepad trigger successfully completes, globally. * @param _event - event.target holds its controller. */ onSqueeze(_event: SelectEvent): void; /** * Called when the controller starts selecting this object the script * represents, e.g. View, ModelView. * @param _event - event.target holds its controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onObjectSelectStart(_event: SelectEvent): boolean | void; /** * Called when the controller stops selecting this object the script * represents, e.g. View, ModelView. * @param _event - event.target holds its controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onObjectSelectEnd(_event: SelectEvent): boolean | void; /** * Called when the controller starts hovering over this object with reticle. * @param _controller - An XR controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onHoverEnter(_controller: THREE.Object3D): boolean | void; /** * Called when the controller hovers over this object with reticle. * @param _controller - An XR controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onHoverExit(_controller: THREE.Object3D): boolean | void; /** * Called when the controller hovers over this object with reticle. * @param _controller - An XR controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onHovering(_controller: THREE.Object3D): boolean | void; /** * Called when a hand's index finger starts touching this object. */ onObjectTouchStart(_event: ObjectTouchEvent): void; /** * Called every frame that a hand's index finger is touching this object. */ onObjectTouching(_event: ObjectTouchEvent): void; /** * Called when a hand's index finger stops touching this object. */ onObjectTouchEnd(_event: ObjectTouchEvent): void; /** * Called when a hand starts grabbing this object (touching + pinching). */ onObjectGrabStart(_event: ObjectGrabEvent): void; /** * Called every frame a hand is grabbing this object. */ onObjectGrabbing(_event: ObjectGrabEvent): void; /** * Called when a hand stops grabbing this object. */ onObjectGrabEnd(_event: ObjectGrabEvent): void; /** * Called when the script is removed from the scene. Opposite of init. */ dispose(): void; readonly isObject3D: true; readonly id: number; uuid: string; name: string; readonly type: string; parent: THREE.Object3D | null; children: THREE.Object3D[]; up: THREE.Vector3; readonly position: THREE.Vector3; readonly rotation: THREE.Euler; readonly quaternion: THREE.Quaternion; readonly scale: THREE.Vector3; readonly modelViewMatrix: THREE.Matrix4; readonly normalMatrix: THREE.Matrix3; matrix: THREE.Matrix4; matrixWorld: THREE.Matrix4; matrixAutoUpdate: boolean; matrixWorldAutoUpdate: boolean; matrixWorldNeedsUpdate: boolean; layers: THREE.Layers; visible: boolean; castShadow: boolean; receiveShadow: boolean; frustumCulled: boolean; renderOrder: number; animations: THREE.AnimationClip[]; userData: Record; customDepthMaterial?: THREE.Material | undefined; customDistanceMaterial?: THREE.Material | undefined; onBeforeShadow(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, shadowCamera: THREE.Camera, geometry: THREE.BufferGeometry, depthMaterial: THREE.Material, group: THREE.Group): void; onAfterShadow(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, shadowCamera: THREE.Camera, geometry: THREE.BufferGeometry, depthMaterial: THREE.Material, group: THREE.Group): void; onBeforeRender(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, geometry: THREE.BufferGeometry, material: THREE.Material, group: THREE.Group): void; onAfterRender(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, geometry: THREE.BufferGeometry, material: THREE.Material, group: THREE.Group): void; applyMatrix4(matrix: THREE.Matrix4): void; applyQuaternion(quaternion: THREE.Quaternion): /*elided*/ any; setRotationFromAxisAngle(axis: THREE.Vector3, angle: number): void; setRotationFromEuler(euler: THREE.Euler): void; setRotationFromMatrix(m: THREE.Matrix4): void; setRotationFromQuaternion(q: THREE.Quaternion): void; rotateOnAxis(axis: THREE.Vector3, angle: number): /*elided*/ any; rotateOnWorldAxis(axis: THREE.Vector3, angle: number): /*elided*/ any; rotateX(angle: number): /*elided*/ any; rotateY(angle: number): /*elided*/ any; rotateZ(angle: number): /*elided*/ any; translateOnAxis(axis: THREE.Vector3, distance: number): /*elided*/ any; translateX(distance: number): /*elided*/ any; translateY(distance: number): /*elided*/ any; translateZ(distance: number): /*elided*/ any; localToWorld(vector: THREE.Vector3): THREE.Vector3; worldToLocal(vector: THREE.Vector3): THREE.Vector3; lookAt(vector: THREE.Vector3): void; lookAt(x: number, y: number, z: number): void; add(...object: THREE.Object3D[]): /*elided*/ any; remove(...object: THREE.Object3D[]): /*elided*/ any; removeFromParent(): /*elided*/ any; clear(): /*elided*/ any; attach(object: THREE.Object3D): /*elided*/ any; getObjectById(id: number): THREE.Object3D | undefined; getObjectByName(name: string): THREE.Object3D | undefined; getObjectByProperty(name: string, value: any): THREE.Object3D | undefined; getObjectsByProperty(name: string, value: any, optionalTarget?: THREE.Object3D[]): THREE.Object3D[]; getWorldPosition(target: THREE.Vector3): THREE.Vector3; getWorldQuaternion(target: THREE.Quaternion): THREE.Quaternion; getWorldScale(target: THREE.Vector3): THREE.Vector3; getWorldDirection(target: THREE.Vector3): THREE.Vector3; raycast(raycaster: THREE.Raycaster, intersects: THREE.Intersection[]): void; traverse(callback: (object: THREE.Object3D) => any): void; traverseVisible(callback: (object: THREE.Object3D) => any): void; traverseAncestors(callback: (object: THREE.Object3D) => any): void; updateMatrix(): void; updateMatrixWorld(force?: boolean): void; updateWorldMatrix(updateParents: boolean, updateChildren: boolean): void; toJSON(meta?: THREE.JSONMeta): THREE.Object3DJSON; clone(recursive?: boolean): /*elided*/ any; copy(object: THREE.Object3D, recursive?: boolean): /*elided*/ any; count?: number | undefined; occlusionTest?: boolean | undefined; static?: boolean | undefined; addEventListener(type: T, listener: THREE.EventListener): void; hasEventListener(type: T, listener: THREE.EventListener): boolean; removeEventListener(type: T, listener: THREE.EventListener): void; dispatchEvent(event: THREE.BaseEvent & THREE.Object3DEventMap[T]): void; }; } & TBase; /** * Script manages app logic or interaction between user and objects. */ declare const ScriptMixinObject3D: { new (...args: any[]): { ux: UX; isXRScript: boolean; /** * Initializes an instance with XR controllers, grips, hands, raycaster, and * default options. We allow all scripts to quickly access its user (e.g., * user.isSelecting(), user.hands), world (e.g., physical depth mesh, * lighting estimation, and recognized objects), and scene (the root of * three.js's scene graph). If this returns a promise, we will wait for it. */ init(_?: object): void | Promise; /** * Runs per frame. */ update(_time?: number, _frame?: XRFrame): void; /** * Enables depth-aware interactions with physics. See /demos/ballpit */ initPhysics(_physics: Physics): void | Promise; physicsStep(): void; onXRSessionStarted(_session?: XRSession): void; onXRSessionEnded(): void; onSimulatorStarted(): void; /** * Called whenever pinch / mouse click starts, globally. * @param _event - event.target holds its controller */ onSelectStart(_event: SelectEvent): void; /** * Called whenever pinch / mouse click discontinues, globally. * @param _event - event.target holds its controller */ onSelectEnd(_event: SelectEvent): void; /** * Called whenever pinch / mouse click successfully completes, globally. * @param _event - event.target holds its controller. */ onSelect(_event: SelectEvent): void; /** * Called whenever pinch / mouse click is happening, globally. */ onSelecting(_event: SelectEvent): void; /** * Called on keyboard keypress. * @param _event - Event containing `.code` to read the keyboard key. */ onKeyDown(_event: KeyEvent): void; onKeyUp(_event: KeyEvent): void; /** * Called whenever gamepad trigger starts, globally. * @param _event - event.target holds its controller. */ onSqueezeStart(_event: SelectEvent): void; /** * Called whenever gamepad trigger stops, globally. * @param _event - event.target holds its controller. */ onSqueezeEnd(_event: SelectEvent): void; /** * Called whenever gamepad is being triggered, globally. */ onSqueezing(_event: SelectEvent): void; /** * Called whenever gamepad trigger successfully completes, globally. * @param _event - event.target holds its controller. */ onSqueeze(_event: SelectEvent): void; /** * Called when the controller starts selecting this object the script * represents, e.g. View, ModelView. * @param _event - event.target holds its controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onObjectSelectStart(_event: SelectEvent): boolean | void; /** * Called when the controller stops selecting this object the script * represents, e.g. View, ModelView. * @param _event - event.target holds its controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onObjectSelectEnd(_event: SelectEvent): boolean | void; /** * Called when the controller starts hovering over this object with reticle. * @param _controller - An XR controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onHoverEnter(_controller: THREE.Object3D): boolean | void; /** * Called when the controller hovers over this object with reticle. * @param _controller - An XR controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onHoverExit(_controller: THREE.Object3D): boolean | void; /** * Called when the controller hovers over this object with reticle. * @param _controller - An XR controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onHovering(_controller: THREE.Object3D): boolean | void; /** * Called when a hand's index finger starts touching this object. */ onObjectTouchStart(_event: ObjectTouchEvent): void; /** * Called every frame that a hand's index finger is touching this object. */ onObjectTouching(_event: ObjectTouchEvent): void; /** * Called when a hand's index finger stops touching this object. */ onObjectTouchEnd(_event: ObjectTouchEvent): void; /** * Called when a hand starts grabbing this object (touching + pinching). */ onObjectGrabStart(_event: ObjectGrabEvent): void; /** * Called every frame a hand is grabbing this object. */ onObjectGrabbing(_event: ObjectGrabEvent): void; /** * Called when a hand stops grabbing this object. */ onObjectGrabEnd(_event: ObjectGrabEvent): void; /** * Called when the script is removed from the scene. Opposite of init. */ dispose(): void; readonly isObject3D: true; readonly id: number; uuid: string; name: string; readonly type: string; parent: THREE.Object3D | null; children: THREE.Object3D[]; up: THREE.Vector3; readonly position: THREE.Vector3; readonly rotation: THREE.Euler; readonly quaternion: THREE.Quaternion; readonly scale: THREE.Vector3; readonly modelViewMatrix: THREE.Matrix4; readonly normalMatrix: THREE.Matrix3; matrix: THREE.Matrix4; matrixWorld: THREE.Matrix4; matrixAutoUpdate: boolean; matrixWorldAutoUpdate: boolean; matrixWorldNeedsUpdate: boolean; layers: THREE.Layers; visible: boolean; castShadow: boolean; receiveShadow: boolean; frustumCulled: boolean; renderOrder: number; animations: THREE.AnimationClip[]; userData: Record; customDepthMaterial?: THREE.Material | undefined; customDistanceMaterial?: THREE.Material | undefined; onBeforeShadow(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, shadowCamera: THREE.Camera, geometry: THREE.BufferGeometry, depthMaterial: THREE.Material, group: THREE.Group): void; onAfterShadow(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, shadowCamera: THREE.Camera, geometry: THREE.BufferGeometry, depthMaterial: THREE.Material, group: THREE.Group): void; onBeforeRender(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, geometry: THREE.BufferGeometry, material: THREE.Material, group: THREE.Group): void; onAfterRender(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, geometry: THREE.BufferGeometry, material: THREE.Material, group: THREE.Group): void; applyMatrix4(matrix: THREE.Matrix4): void; applyQuaternion(quaternion: THREE.Quaternion): /*elided*/ any; setRotationFromAxisAngle(axis: THREE.Vector3, angle: number): void; setRotationFromEuler(euler: THREE.Euler): void; setRotationFromMatrix(m: THREE.Matrix4): void; setRotationFromQuaternion(q: THREE.Quaternion): void; rotateOnAxis(axis: THREE.Vector3, angle: number): /*elided*/ any; rotateOnWorldAxis(axis: THREE.Vector3, angle: number): /*elided*/ any; rotateX(angle: number): /*elided*/ any; rotateY(angle: number): /*elided*/ any; rotateZ(angle: number): /*elided*/ any; translateOnAxis(axis: THREE.Vector3, distance: number): /*elided*/ any; translateX(distance: number): /*elided*/ any; translateY(distance: number): /*elided*/ any; translateZ(distance: number): /*elided*/ any; localToWorld(vector: THREE.Vector3): THREE.Vector3; worldToLocal(vector: THREE.Vector3): THREE.Vector3; lookAt(vector: THREE.Vector3): void; lookAt(x: number, y: number, z: number): void; add(...object: THREE.Object3D[]): /*elided*/ any; remove(...object: THREE.Object3D[]): /*elided*/ any; removeFromParent(): /*elided*/ any; clear(): /*elided*/ any; attach(object: THREE.Object3D): /*elided*/ any; getObjectById(id: number): THREE.Object3D | undefined; getObjectByName(name: string): THREE.Object3D | undefined; getObjectByProperty(name: string, value: any): THREE.Object3D | undefined; getObjectsByProperty(name: string, value: any, optionalTarget?: THREE.Object3D[]): THREE.Object3D[]; getWorldPosition(target: THREE.Vector3): THREE.Vector3; getWorldQuaternion(target: THREE.Quaternion): THREE.Quaternion; getWorldScale(target: THREE.Vector3): THREE.Vector3; getWorldDirection(target: THREE.Vector3): THREE.Vector3; raycast(raycaster: THREE.Raycaster, intersects: THREE.Intersection[]): void; traverse(callback: (object: THREE.Object3D) => any): void; traverseVisible(callback: (object: THREE.Object3D) => any): void; traverseAncestors(callback: (object: THREE.Object3D) => any): void; updateMatrix(): void; updateMatrixWorld(force?: boolean): void; updateWorldMatrix(updateParents: boolean, updateChildren: boolean): void; toJSON(meta?: THREE.JSONMeta): THREE.Object3DJSON; clone(recursive?: boolean): /*elided*/ any; copy(object: THREE.Object3D, recursive?: boolean): /*elided*/ any; count?: number | undefined; occlusionTest?: boolean | undefined; static?: boolean | undefined; addEventListener(type: T, listener: THREE.EventListener): void; hasEventListener(type: T, listener: THREE.EventListener): boolean; removeEventListener(type: T, listener: THREE.EventListener): void; dispatchEvent(event: THREE.BaseEvent & THREE.Object3DEventMap[T]): void; }; } & typeof THREE.Object3D; declare class Script extends ScriptMixinObject3D { } /** * MeshScript can be constructed with geometry and materials, with * `super(geometry, material)`; for direct access to its geometry. * MeshScripts hold a UX object that contains its interaction information such * as which controller is selecting or touching this object, as well as the * exact selected UV / xyz of the reticle, or touched point. */ declare const ScriptMixinMeshScript: { new (...args: any[]): { ux: UX; isXRScript: boolean; /** * Initializes an instance with XR controllers, grips, hands, raycaster, and * default options. We allow all scripts to quickly access its user (e.g., * user.isSelecting(), user.hands), world (e.g., physical depth mesh, * lighting estimation, and recognized objects), and scene (the root of * three.js's scene graph). If this returns a promise, we will wait for it. */ init(_?: object): void | Promise; /** * Runs per frame. */ update(_time?: number, _frame?: XRFrame): void; /** * Enables depth-aware interactions with physics. See /demos/ballpit */ initPhysics(_physics: Physics): void | Promise; physicsStep(): void; onXRSessionStarted(_session?: XRSession): void; onXRSessionEnded(): void; onSimulatorStarted(): void; /** * Called whenever pinch / mouse click starts, globally. * @param _event - event.target holds its controller */ onSelectStart(_event: SelectEvent): void; /** * Called whenever pinch / mouse click discontinues, globally. * @param _event - event.target holds its controller */ onSelectEnd(_event: SelectEvent): void; /** * Called whenever pinch / mouse click successfully completes, globally. * @param _event - event.target holds its controller. */ onSelect(_event: SelectEvent): void; /** * Called whenever pinch / mouse click is happening, globally. */ onSelecting(_event: SelectEvent): void; /** * Called on keyboard keypress. * @param _event - Event containing `.code` to read the keyboard key. */ onKeyDown(_event: KeyEvent): void; onKeyUp(_event: KeyEvent): void; /** * Called whenever gamepad trigger starts, globally. * @param _event - event.target holds its controller. */ onSqueezeStart(_event: SelectEvent): void; /** * Called whenever gamepad trigger stops, globally. * @param _event - event.target holds its controller. */ onSqueezeEnd(_event: SelectEvent): void; /** * Called whenever gamepad is being triggered, globally. */ onSqueezing(_event: SelectEvent): void; /** * Called whenever gamepad trigger successfully completes, globally. * @param _event - event.target holds its controller. */ onSqueeze(_event: SelectEvent): void; /** * Called when the controller starts selecting this object the script * represents, e.g. View, ModelView. * @param _event - event.target holds its controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onObjectSelectStart(_event: SelectEvent): boolean | void; /** * Called when the controller stops selecting this object the script * represents, e.g. View, ModelView. * @param _event - event.target holds its controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onObjectSelectEnd(_event: SelectEvent): boolean | void; /** * Called when the controller starts hovering over this object with reticle. * @param _controller - An XR controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onHoverEnter(_controller: THREE.Object3D): boolean | void; /** * Called when the controller hovers over this object with reticle. * @param _controller - An XR controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onHoverExit(_controller: THREE.Object3D): boolean | void; /** * Called when the controller hovers over this object with reticle. * @param _controller - An XR controller. * @returns Whether the event was handled. If true, the event will not bubble up. */ onHovering(_controller: THREE.Object3D): boolean | void; /** * Called when a hand's index finger starts touching this object. */ onObjectTouchStart(_event: ObjectTouchEvent): void; /** * Called every frame that a hand's index finger is touching this object. */ onObjectTouching(_event: ObjectTouchEvent): void; /** * Called when a hand's index finger stops touching this object. */ onObjectTouchEnd(_event: ObjectTouchEvent): void; /** * Called when a hand starts grabbing this object (touching + pinching). */ onObjectGrabStart(_event: ObjectGrabEvent): void; /** * Called every frame a hand is grabbing this object. */ onObjectGrabbing(_event: ObjectGrabEvent): void; /** * Called when a hand stops grabbing this object. */ onObjectGrabEnd(_event: ObjectGrabEvent): void; /** * Called when the script is removed from the scene. Opposite of init. */ dispose(): void; readonly isObject3D: true; readonly id: number; uuid: string; name: string; readonly type: string; parent: THREE.Object3D | null; children: THREE.Object3D[]; up: THREE.Vector3; readonly position: THREE.Vector3; readonly rotation: THREE.Euler; readonly quaternion: THREE.Quaternion; readonly scale: THREE.Vector3; readonly modelViewMatrix: THREE.Matrix4; readonly normalMatrix: THREE.Matrix3; matrix: THREE.Matrix4; matrixWorld: THREE.Matrix4; matrixAutoUpdate: boolean; matrixWorldAutoUpdate: boolean; matrixWorldNeedsUpdate: boolean; layers: THREE.Layers; visible: boolean; castShadow: boolean; receiveShadow: boolean; frustumCulled: boolean; renderOrder: number; animations: THREE.AnimationClip[]; userData: Record; customDepthMaterial?: THREE.Material | undefined; customDistanceMaterial?: THREE.Material | undefined; onBeforeShadow(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, shadowCamera: THREE.Camera, geometry: THREE.BufferGeometry, depthMaterial: THREE.Material, group: THREE.Group): void; onAfterShadow(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, shadowCamera: THREE.Camera, geometry: THREE.BufferGeometry, depthMaterial: THREE.Material, group: THREE.Group): void; onBeforeRender(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, geometry: THREE.BufferGeometry, material: THREE.Material, group: THREE.Group): void; onAfterRender(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera, geometry: THREE.BufferGeometry, material: THREE.Material, group: THREE.Group): void; applyMatrix4(matrix: THREE.Matrix4): void; applyQuaternion(quaternion: THREE.Quaternion): /*elided*/ any; setRotationFromAxisAngle(axis: THREE.Vector3, angle: number): void; setRotationFromEuler(euler: THREE.Euler): void; setRotationFromMatrix(m: THREE.Matrix4): void; setRotationFromQuaternion(q: THREE.Quaternion): void; rotateOnAxis(axis: THREE.Vector3, angle: number): /*elided*/ any; rotateOnWorldAxis(axis: THREE.Vector3, angle: number): /*elided*/ any; rotateX(angle: number): /*elided*/ any; rotateY(angle: number): /*elided*/ any; rotateZ(angle: number): /*elided*/ any; translateOnAxis(axis: THREE.Vector3, distance: number): /*elided*/ any; translateX(distance: number): /*elided*/ any; translateY(distance: number): /*elided*/ any; translateZ(distance: number): /*elided*/ any; localToWorld(vector: THREE.Vector3): THREE.Vector3; worldToLocal(vector: THREE.Vector3): THREE.Vector3; lookAt(vector: THREE.Vector3): void; lookAt(x: number, y: number, z: number): void; add(...object: THREE.Object3D[]): /*elided*/ any; remove(...object: THREE.Object3D[]): /*elided*/ any; removeFromParent(): /*elided*/ any; clear(): /*elided*/ any; attach(object: THREE.Object3D): /*elided*/ any; getObjectById(id: number): THREE.Object3D | undefined; getObjectByName(name: string): THREE.Object3D | undefined; getObjectByProperty(name: string, value: any): THREE.Object3D | undefined; getObjectsByProperty(name: string, value: any, optionalTarget?: THREE.Object3D[]): THREE.Object3D[]; getWorldPosition(target: THREE.Vector3): THREE.Vector3; getWorldQuaternion(target: THREE.Quaternion): THREE.Quaternion; getWorldScale(target: THREE.Vector3): THREE.Vector3; getWorldDirection(target: THREE.Vector3): THREE.Vector3; raycast(raycaster: THREE.Raycaster, intersects: THREE.Intersection[]): void; traverse(callback: (object: THREE.Object3D) => any): void; traverseVisible(callback: (object: THREE.Object3D) => any): void; traverseAncestors(callback: (object: THREE.Object3D) => any): void; updateMatrix(): void; updateMatrixWorld(force?: boolean): void; updateWorldMatrix(updateParents: boolean, updateChildren: boolean): void; toJSON(meta?: THREE.JSONMeta): THREE.Object3DJSON; clone(recursive?: boolean): /*elided*/ any; copy(object: THREE.Object3D, recursive?: boolean): /*elided*/ any; count?: number | undefined; occlusionTest?: boolean | undefined; static?: boolean | undefined; addEventListener(type: T, listener: THREE.EventListener): void; hasEventListener(type: T, listener: THREE.EventListener): boolean; removeEventListener(type: T, listener: THREE.EventListener): void; dispatchEvent(event: THREE.BaseEvent & THREE.Object3DEventMap[T]): void; }; } & typeof THREE.Mesh; declare class MeshScript extends ScriptMixinMeshScript { /** * {@inheritDoc} */ constructor(geometry?: TGeometry, material?: TMaterial); } declare const GEMINI_DEFAULT_FLASH_MODEL = "gemini-2.5-flash"; declare const GEMINI_DEFAULT_LIVE_MODEL = "gemini-2.5-flash-native-audio-preview-12-2025"; declare class GeminiOptions { apiKey: string; urlParam: string; keyValid: boolean; enabled: boolean; model: string; liveModel: string; config: GoogleGenAITypes.GenerateContentConfig; } declare class OpenAIOptions { apiKey: string; urlParam: string; model: string; enabled: boolean; } type AIModel = 'gemini' | 'openai'; declare class AIOptions { enabled: boolean; model: AIModel; gemini: GeminiOptions; openai: OpenAIOptions; globalUrlParams: { key: string; }; } interface ToolCall { name: string; args: unknown; } /** * Standardized result type for tool execution. * @typeParam T - The type of data returned on success. */ interface ToolResult { /** Whether the tool execution succeeded */ success: boolean; /** The result data if successful */ data?: T; /** Error message if execution failed */ error?: string; /** Additional metadata about the execution */ metadata?: Record; } type ToolSchema = Omit & { properties?: Record; type?: keyof typeof GoogleGenAITypes.Type; }; type ToolOptions = { /** The name of the tool. */ name: string; /** A description of what the tool does. */ description: string; /** The parameters of the tool */ parameters?: ToolSchema; /** A callback to execute when the tool is triggered */ onTriggered?: (args: unknown) => unknown | Promise; behavior?: 'BLOCKING' | 'NON_BLOCKING' | GoogleGenAITypes.Behavior; }; /** * A base class for tools that the agent can use. */ declare class Tool { name: string; description?: string; parameters?: ToolSchema; onTriggered?: (args: unknown) => unknown; behavior?: 'BLOCKING' | 'NON_BLOCKING'; /** * @param options - The options for the tool. */ constructor(options: ToolOptions); /** * Executes the tool's action with standardized error handling. * @param args - The arguments for the tool. * @returns A promise that resolves with a ToolResult containing success/error information. */ execute(args: unknown): Promise; /** * Returns a JSON representation of the tool. * @returns A valid FunctionDeclaration object. */ toJSON(): GoogleGenAITypes.FunctionDeclaration; } interface GeminiResponse { toolCall?: ToolCall; text?: string | null; } declare abstract class BaseAIModel { constructor(); abstract init(): Promise; abstract isAvailable(): boolean; abstract query(_input: object, _tools: []): Promise; hasApiKey(): Promise; } interface GeminiQueryInput { type: 'live' | 'text' | 'uri' | 'base64' | 'multiPart'; action?: 'start' | 'stop' | 'send'; text?: string; uri?: string; base64?: string; mimeType?: string; parts?: GoogleGenAITypes.Part[]; config?: GoogleGenAITypes.LiveConnectConfig; data?: GoogleGenAITypes.LiveSendRealtimeInputParameters; useExponentialBackoff?: boolean; } declare class Gemini extends BaseAIModel { protected options: GeminiOptions; inited: boolean; liveSession?: GoogleGenAITypes.Session; isLiveMode: boolean; liveCallbacks: Partial; ai?: GoogleGenAITypes.GoogleGenAI; constructor(options: GeminiOptions); init(): Promise; isAvailable(): boolean; isLiveAvailable(): false | typeof GoogleGenAITypes.Modality | undefined; startLiveSession(params?: GoogleGenAITypes.LiveConnectConfig, model?: string): Promise; stopLiveSession(): Promise; setLiveCallbacks(callbacks: GoogleGenAITypes.LiveCallbacks): void; sendToolResponse(response: GoogleGenAITypes.LiveSendToolResponseParameters): void; sendRealtimeInput(input: GoogleGenAITypes.LiveSendRealtimeInputParameters): void; getLiveSessionStatus(): { isActive: boolean; hasSession: boolean; isAvailable: boolean | typeof GoogleGenAITypes.Modality | undefined; }; query(input: GeminiQueryInput | { prompt: string; }): Promise; protected queryOnce(input: GeminiQueryInput | { prompt: string; }): Promise; protected queryWithExponentialFalloff(input: GeminiQueryInput | { prompt: string; }): Promise; generate(prompt: string | string[], type?: 'image', systemInstruction?: string, model?: string): Promise; hasApiKey(): Promise; } declare class OpenAI extends BaseAIModel { protected options: OpenAIOptions; openai?: OpenAIType; constructor(options: OpenAIOptions); init(): Promise; isAvailable(): boolean; query(input: { prompt: string; }, _tools?: never[]): Promise<{ text: string; } | null>; generate(): Promise; } type ModelClass = Gemini | OpenAI; type ModelOptions = GeminiOptions | OpenAIOptions; type KeysJson = { [key: string]: string | { apiKey?: string; }; }; /** * AI Interface to wrap different AI models (primarily Gemini) * Handles both traditional query-based AI interactions and real-time live * sessions * * Features: * - Text and multimodal queries * - Real-time audio/video AI sessions (Gemini Live) * - Advanced API key management with multiple sources * - Session locking to prevent concurrent operations * * The URL param and key.json shortcut is only for demonstration and prototyping * practice and we strongly suggest not using it for production or deployment * purposes. One should set up a proper server to converse with AI servers in * deployment. * * API Key Management Features: * * 1. Multiple Key Sources (Priority Order): * - URL Parameter: ?key=\ * - keys.json file: Local configuration file * - User Prompt: Interactive fallback * 2. keys.json Support: * - Structure: \{"gemini": \{"apiKey": "YOUR_KEY_HERE"\}\} * - Automatically loads if present */ declare class AI extends Script { static dependencies: { aiOptions: typeof AIOptions; }; editorIcon: string; model?: ModelClass; lock: boolean; options: AIOptions; keysCache?: KeysJson; /** * Load API keys from keys.json file if available * Parsed keys object or null if not found */ loadKeysFromFile(): Promise; init({ aiOptions }: { aiOptions: AIOptions; }): Promise; initializeModel(ModelClass: typeof Gemini | typeof OpenAI, modelOptions: ModelOptions): Promise; resolveApiKey(modelOptions: ModelOptions): Promise; isValidApiKey(key: string): boolean | ""; isAvailable(): boolean | undefined; query(input: { prompt: string; }, tools?: never[]): Promise; startLiveSession(config?: GoogleGenAITypes.LiveConnectConfig, model?: string): Promise; stopLiveSession(): Promise; setLiveCallbacks(callbacks: GoogleGenAITypes.LiveCallbacks): Promise; sendToolResponse(response: GoogleGenAITypes.LiveSendToolResponseParameters): void; sendRealtimeInput(input: GoogleGenAITypes.LiveSendRealtimeInputParameters): false | void; getLiveSessionStatus(): { isActive: boolean; hasSession: boolean; isAvailable: boolean | typeof GoogleGenAITypes.Modality | undefined; }; isLiveAvailable(): false | typeof GoogleGenAITypes.Modality | undefined; /** * In simulator mode, pop up a 2D UI to request Gemini key; * In XR mode, show a 3D UI to instruct users to get an API key. */ triggerKeyPopup(): void; generate(prompt: string | string[], type?: 'image', systemInstruction?: string, model?: undefined): Promise; /** * Create a sample keys.json file structure for reference * @returns Sample keys.json structure */ static createSampleKeysStructure(): { gemini: { apiKey: string; }; openai: { apiKey: string; }; }; /** * Check if the current model has an API key available from any source * @returns True if API key is available */ hasApiKey(): Promise; } interface MemoryEntry { role: 'user' | 'ai' | 'tool'; content: string; } /** * Manages the agent's memory, including short-term, long-term, and working * memory. */ declare class Memory { private shortTermMemory; /** * Adds a new entry to the short-term memory. * @param entry - The memory entry to add. */ addShortTerm(entry: MemoryEntry): void; /** * Retrieves the short-term memory. * @returns An array of all short-term memory entries. */ getShortTerm(): MemoryEntry[]; /** * Clears all memory components. */ clear(): void; } /** * Builds the context to be sent to the AI for reasoning. */ declare class Context { private instructions; constructor(instructions?: string); get instruction(): string; /** * Constructs a formatted prompt from memory and available tools. * @param memory - The agent's memory. * @param tools - The list of available tools. * @returns A string representing the full context for the AI. */ build(memory: Memory, tools: Tool[]): string; private formatEntry; } /** * Lifecycle callbacks for agent events. */ interface AgentLifecycleCallbacks { /** Called when a session starts */ onSessionStart?: () => void | Promise; /** Called when a session ends */ onSessionEnd?: () => void | Promise; /** Called after a tool is executed */ onToolExecuted?: (toolName: string, result: unknown) => void; /** Called when an error occurs */ onError?: (error: Error) => void; } /** * An agent that can use an AI to reason and execute tools. */ declare class Agent { static dependencies: {}; ai: AI; tools: Tool[]; memory: Memory; contextBuilder: Context; lifecycleCallbacks?: AgentLifecycleCallbacks; isSessionActive: boolean; constructor(ai: AI, tools?: Tool[], instruction?: string, callbacks?: AgentLifecycleCallbacks); /** * Starts the agent's reasoning loop with an initial prompt. * @param prompt - The initial prompt from the user. * @returns The final text response from the agent. */ start(prompt: string): Promise; /** * The main reasoning and action loop of the agent for non-live mode. * It repeatedly builds context, queries the AI, and executes tools * until a final text response is generated. */ private run; findTool(name: string): Tool | undefined; /** * Get the current session state. * @returns Object containing session information */ getSessionState(): { isActive: boolean; toolCount: number; memorySize: number; }; } declare class Registry { private instances; /** * Registers an new instanceof a given type. * If an existing instance of the same type is already registered, it will be * overwritten. * @param instance - The instance to register. * @param type - Type to register the instance as. Will default to * `instance.constructor` if not defined. */ register(instance: T, type?: Constructor): void; /** * Gets an existing instance of a registered type. * @param type - The constructor function of the type to retrieve. * @returns The instance of the requested type. */ get(type: Constructor): T | undefined; /** * Gets an existing instance of a registered type, or creates a new one if it * doesn't exist. * @param type - The constructor function of the type to retrieve. * @param factory - A function that creates a new instance of the type if it * doesn't already exist. * @returns The instance of the requested type. */ getOrCreate(type: Constructor, factory: () => T): T; /** * Unregisters an instance of a given type. * @param type - The type to unregister. */ unregister(type: Constructor): void; } interface AudioListenerOptions { sampleRate?: number; channelCount?: number; echoCancellation?: boolean; noiseSuppression?: boolean; autoGainControl?: boolean; } declare class AudioListener extends Script { static dependencies: { registry: typeof Registry; }; private options; private audioStream?; audioContext?: AudioContext; private sourceNode?; private processorNode?; private isCapturing; private latestAudioBuffer; private accumulatedChunks; private isAccumulating; private registry; aiService?: AI; private onAudioData?; private onError?; constructor(options?: AudioListenerOptions); /** * Init the AudioListener. */ init({ registry }: { registry: Registry; }): void; startCapture(callbacks?: { onAudioData?: (audioBuffer: ArrayBuffer) => void; onError?: (error: Error) => void; accumulate?: boolean; }): Promise; stopCapture(): void; setupAudioCapture(): Promise; private setupAudioWorklet; streamToAI(audioBuffer: ArrayBuffer): void; setAIStreaming(enabled: boolean): void; cleanup(): void; static isSupported(): boolean; getIsCapturing(): boolean; getLatestAudioBuffer(): ArrayBuffer | null; clearLatestAudioBuffer(): void; /** * Gets all accumulated audio chunks as a single combined buffer */ getAccumulatedBuffer(): ArrayBuffer | null; /** * Clears accumulated chunks */ clearAccumulatedBuffer(): void; /** * Gets the number of accumulated chunks */ getAccumulatedChunkCount(): number; dispose(): void; } declare enum VolumeCategory { music = "music", sfx = "sfx", speech = "speech", ui = "ui" } declare class CategoryVolumes { isMuted: boolean; masterVolume: number; volumes: Record; getCategoryVolume(category: string): number; getEffectiveVolume(category: string, specificVolume?: number): number; } interface AudioPlayerOptions { sampleRate?: number; channelCount?: number; category?: string; } declare class AudioPlayer extends Script { private options; private audioContext?; private audioQueue; private nextStartTime; private gainNode?; private categoryVolumes?; private volume; private category; scheduleAheadTime: number; constructor(options?: AudioPlayerOptions); /** * Sets the CategoryVolumes instance for this player to respect * master/category volumes */ setCategoryVolumes(categoryVolumes: CategoryVolumes): void; /** * Sets the specific volume for this player (0.0 to 1.0) */ setVolume(level: number): void; /** * Updates the gain node volume based on category volumes * Public so CoreSound can update it when master volume changes */ updateGainNodeVolume(): void; initializeAudioContext(): Promise; playAudioChunk(base64AudioData: string): Promise; private scheduleAudioBuffers; clearQueue(): void; getIsPlaying(): boolean; getQueueLength(): number; base64ToArrayBuffer(base64: string): ArrayBuffer; stop(): void; static isSupported(): boolean; dispose(): void; } declare const musicLibrary: { readonly ambient: string; readonly background: string; readonly buttonHover: string; readonly buttonPress: string; readonly menuDismiss: string; }; declare class BackgroundMusic extends Script { private listener; private categoryVolumes; private audioLoader; private currentAudio; private isPlaying; private musicLibrary; private specificVolume; private musicCategory; constructor(listener: THREE.AudioListener, categoryVolumes: CategoryVolumes); setVolume(level: number): void; playMusic(musicKey: keyof typeof musicLibrary, category?: string): void; stopMusic(): void; destroy(): void; } declare class SpeechSynthesizerOptions { enabled: boolean; /** If true, a new call to speak() will interrupt any ongoing speech. */ allowInterruptions: boolean; } declare class SpeechRecognizerOptions { enabled: boolean; /** Recognition language (e.g., 'en-US'). */ lang: string; /** If true, recognition continues after a pause. */ continuous: boolean; /** Keywords to detect as commands. */ commands: string[]; /** If true, provides interim results. */ interimResults: boolean; /** Minimum confidence (0-1) for a command. */ commandConfidenceThreshold: number; /** If true, play activation sounds in simulator. */ playSimulatorActivationSounds: boolean; } declare class SoundOptions { speechSynthesizer: SpeechSynthesizerOptions; speechRecognizer: SpeechRecognizerOptions; } /** * Defines common UI sound presets with their default parameters. * Each preset specifies frequency, duration, and waveform type. */ declare const SOUND_PRESETS: { readonly BEEP: { readonly frequency: 1000; readonly duration: 0.07; readonly waveformType: "sine"; }; readonly CLICK: readonly [{ readonly frequency: 1500; readonly duration: 0.02; readonly waveformType: "triangle"; readonly delay: 0; }]; readonly ACTIVATE: readonly [{ readonly frequency: 800; readonly duration: 0.05; readonly waveformType: "sine"; readonly delay: 0; }, { readonly frequency: 1200; readonly duration: 0.07; readonly waveformType: "sine"; readonly delay: 50; }]; readonly DEACTIVATE: readonly [{ readonly frequency: 1200; readonly duration: 0.05; readonly waveformType: "sine"; readonly delay: 0; }, { readonly frequency: 800; readonly duration: 0.07; readonly waveformType: "sine"; readonly delay: 50; }]; }; declare class SoundSynthesizer extends Script { audioContext?: AudioContext; isInitialized: boolean; debug: boolean; /** * Initializes the AudioContext. */ private _initAudioContext; /** * Plays a single tone with specified parameters. * @param frequency - The frequency of the tone in Hz. * @param duration - The duration of the tone in seconds. * @param volume - The volume of the tone (0.0 to 1.0). * @param waveformType - The type of waveform ('sine', 'square', 'sawtooth', * 'triangle'). */ playTone(frequency: number, duration: number, volume: number, waveformType: OscillatorType): void; /** * Plays a predefined sound preset. * @param presetName - The name of the preset (e.g., 'BEEP', 'CLICK', * 'ACTIVATE', 'DEACTIVATE'). * @param volume - The volume for the preset (overrides default * if present, otherwise uses this). */ playPresetTone(presetName: keyof typeof SOUND_PRESETS, volume?: number): void; } declare const spatialSoundLibrary: { readonly ambient: "musicLibrary/AmbientLoop.opus"; readonly buttonHover: "musicLibrary/ButtonHover.opus"; readonly paintOneShot1: "musicLibrary/PaintOneShot1.opus"; }; interface PlaySoundOptions { loop?: boolean; volume?: number; refDistance?: number; rolloffFactor?: number; onEnded?: () => void; } declare class SpatialAudio extends Script { private listener; private categoryVolumes; private audioLoader; private soundLibrary; private activeSounds; private specificVolume; private category; private defaultRefDistance; private defaultRolloffFactor; constructor(listener: THREE.AudioListener, categoryVolumes: CategoryVolumes); /** * Plays a sound attached to a specific 3D object. * @param soundKey - Key from the soundLibrary. * @param targetObject - The object the sound should emanate * from. * @param options - Optional settings \{ loop: boolean, volume: * number, refDistance: number, rolloffFactor: number, onEnded: function * \}. * @returns A unique ID for the playing sound instance, or null * if failed. */ playSoundAtObject(soundKey: keyof typeof spatialSoundLibrary, targetObject: THREE.Object3D, options?: PlaySoundOptions): number | null; /** * Stops a specific sound instance by its ID. * @param soundId - The ID returned by playSoundAtObject. */ stopSound(soundId: number): void; /** * Internal method to remove sound from object and map. * @param soundId - id */ private _cleanupSound; /** * Sets the base specific volume for subsequently played spatial sounds. * Does NOT affect currently playing sounds (use updateAllVolumes for that). * @param level - Volume level (0.0 to 1.0). */ setVolume(level: number): void; /** * Updates the volume of all currently playing spatial sounds managed by this * instance. */ updateAllVolumes(): void; destroy(): void; } interface SpeechRecognizerEventMap extends THREE.Object3DEventMap { start: object; error: { error: string; }; end: object; result: { originalEvent: SpeechRecognitionEvent; transcript: string; confidence: number; command?: string; isFinal: boolean; }; } declare class SpeechRecognizer extends Script { private soundSynthesizer; static dependencies: { soundOptions: typeof SoundOptions; }; options: SpeechRecognizerOptions; recognition?: SpeechRecognition; isListening: boolean; lastTranscript: string; lastCommand?: string; lastConfidence: number; error?: string; playActivationSounds: boolean; private handleStartBound; private handleResultBound; private handleEndBound; private handleErrorBound; constructor(soundSynthesizer: SoundSynthesizer); init({ soundOptions }: { soundOptions: SoundOptions; }): void; onSimulatorStarted(): void; start(): void; stop(): void; getLastTranscript(): string; getLastCommand(): string | undefined; getLastConfidence(): number; private _handleStart; private _handleResult; _handleEnd(): void; _handleError(event: SpeechRecognitionErrorEvent): void; destroy(): void; } declare class SpeechSynthesizer extends Script { private categoryVolumes; private onStartCallback; private onEndCallback; private onErrorCallback; static dependencies: { soundOptions: typeof SoundOptions; }; private synth; private voices; private selectedVoice?; private isSpeaking; private debug; private specificVolume; private speechCategory; private options; constructor(categoryVolumes: CategoryVolumes, onStartCallback?: () => void, onEndCallback?: () => void, onErrorCallback?: (_: Error) => void); init({ soundOptions }: { soundOptions: SoundOptions; }): void; loadVoices(): void; setVolume(level: number): void; speak(text: string, lang?: string, pitch?: number, rate?: number): Promise; tts(text: string, lang?: string, pitch?: number, rate?: number): void; cancel(): void; destroy(): void; } declare class CoreSound extends Script { static dependencies: { camera: typeof THREE.Camera; soundOptions: typeof SoundOptions; }; type: string; name: string; categoryVolumes: CategoryVolumes; soundSynthesizer: SoundSynthesizer; listener: THREE.AudioListener; backgroundMusic: BackgroundMusic; spatialAudio: SpatialAudio; speechRecognizer?: SpeechRecognizer; speechSynthesizer?: SpeechSynthesizer; audioListener: AudioListener; audioPlayer: AudioPlayer; options: SoundOptions; init({ camera, soundOptions, }: { camera: THREE.Camera; soundOptions: SoundOptions; }): void; getAudioListener(): THREE.AudioListener; setMasterVolume(level: number): void; getMasterVolume(): number; setCategoryVolume(category: VolumeCategory, level: number): void; getCategoryVolume(category: VolumeCategory): number; enableAudio(options?: { streamToAI?: boolean; accumulate?: boolean; }): Promise; disableAudio(): void; /** * Starts recording audio with chunk accumulation */ startRecording(): Promise; /** * Stops recording and returns the accumulated audio buffer */ stopRecording(): ArrayBuffer | null; /** * Gets the accumulated recording buffer without stopping */ getRecordedBuffer(): ArrayBuffer | null; /** * Clears the accumulated recording buffer */ clearRecordedBuffer(): void; /** * Gets the sample rate being used for recording */ getRecordingSampleRate(): number; setAIStreaming(enabled: boolean): void; isAIStreamingEnabled(): boolean; playAIAudio(base64AudioData: string): Promise; stopAIAudio(): void; isAIAudioPlaying(): boolean; /** * Plays a raw audio buffer (Int16 PCM data) with proper sample rate */ playRecordedAudio(audioBuffer: ArrayBuffer, sampleRate?: number): Promise; isAudioEnabled(): boolean; getLatestAudioBuffer(): ArrayBuffer | null; clearLatestAudioBuffer(): void; getEffectiveVolume(category: VolumeCategory, specificVolume?: number): number; muteAll(): void; unmuteAll(): void; destroy(): void; } /** * State information for a live session. */ interface LiveSessionState { /** Whether the session is currently active */ isActive: boolean; /** Timestamp when session started */ startTime?: number; /** Timestamp when session ended */ endTime?: number; /** Number of messages received */ messageCount: number; /** Number of tool calls executed */ toolCallCount: number; /** Last error message if any */ lastError?: string; } /** * Skybox Agent for generating 360-degree equirectangular backgrounds through conversation. * * @example Basic usage * ```typescript * // 1. Enable audio (required for live sessions) * await xb.core.sound.enableAudio(); * * // 2. Create agent * const agent = new xb.SkyboxAgent(xb.core.ai, xb.core.sound, xb.core.scene); * * // 3. Start session * await agent.startLiveSession({ * onopen: () => console.log('Session ready'), * onmessage: (msg) => handleMessage(msg), * onclose: () => console.log('Session closed') * }); * * // 4. Clean up when done * await agent.stopLiveSession(); * xb.core.sound.disableAudio(); * ``` * * @example With lifecycle callbacks * ```typescript * const agent = new xb.SkyboxAgent( * xb.core.ai, * xb.core.sound, * xb.core.scene, * { * onSessionStart: () => updateUI('active'), * onSessionEnd: () => updateUI('inactive'), * onError: (error) => showError(error) * } * ); * ``` * * @remarks * - Audio must be enabled BEFORE starting live session using `xb.core.sound.enableAudio()` * - Users are responsible for managing audio lifecycle * - Always call `stopLiveSession()` before disabling audio * - Session state can be checked using `getSessionState()` and `getLiveSessionState()` */ declare class SkyboxAgent extends Agent { private sound; private sessionState; constructor(ai: AI, sound: CoreSound, scene: THREE.Scene, callbacks?: AgentLifecycleCallbacks); /** * Starts a live AI session for real-time conversation. * * @param callbacks - Optional callbacks for session events. Can also be set using ai.setLiveCallbacks() * @throws If AI model is not initialized or live session is not available * * @remarks * Audio must be enabled separately using `xb.core.sound.enableAudio()` before starting the session. * This gives users control over when microphone permissions are requested. */ startLiveSession(callbacks?: GoogleGenAITypes.LiveCallbacks): Promise; /** * Stops the live AI session. * * @remarks * Audio must be disabled separately using `xb.core.sound.disableAudio()` after stopping the session. */ stopLiveSession(): Promise; /** * Wraps user callbacks to track session state and trigger lifecycle events. * @param callbacks - The callbacks to wrap. * @returns The wrapped callbacks. */ private wrapCallbacks; /** * Sends tool execution results back to the AI. * * @param response - The tool response containing function results */ sendToolResponse(response: GoogleGenAITypes.LiveSendToolResponseParameters): Promise; /** * Validates that a tool response has the correct format. * @param response - The tool response to validate. * @returns True if the response is valid, false otherwise. */ private validateToolResponse; /** * Helper to create a properly formatted tool response from a ToolResult. * * @param id - The function call ID * @param name - The function name * @param result - The ToolResult from tool execution * @returns A properly formatted FunctionResponse */ static createToolResponse(id: string, name: string, result: ToolResult): GoogleGenAITypes.FunctionResponse; /** * Gets the current live session state. * * @returns Read-only session state information */ getLiveSessionState(): Readonly; /** * Gets the duration of the session in milliseconds. * * @returns Duration in ms, or null if session hasn't started */ getSessionDuration(): number | null; } interface GetWeatherArgs { latitude: number; longitude: number; } interface WeatherData { temperature: number; weathercode: number; } /** * A tool that gets the current weather for a specific location. */ declare class GetWeatherTool extends Tool { constructor(); /** * Executes the tool's action. * @param args - The arguments for the tool. * @returns A promise that resolves with a ToolResult containing weather information. */ execute(args: GetWeatherArgs): Promise>; } /** * A tool that generates a 360-degree equirectangular skybox image * based on a given prompt using an AI service. */ declare class GenerateSkyboxTool extends Tool { private ai; private scene; constructor(ai: AI, scene: THREE.Scene); /** * Executes the tool's action. * @param args - The prompt to use to generate the skybox. * @returns A promise that resolves with a ToolResult containing success/error information. */ execute(args: { prompt: string; }): Promise>; } /** * Parameters for RGB to depth UV mapping given different aspect ratios. * These parameters define the distortion model and affine transformations * required to align the RGB camera feed with the depth map. */ interface RgbToDepthParams { scale: number; scaleX: number; scaleY: number; translateU: number; translateV: number; k1: number; k2: number; k3: number; p1: number; p2: number; xc: number; yc: number; } /** * Default parameters for rgb to depth projection. * For RGB and depth, 4:3 and 1:1, respectively. */ declare const DEFAULT_RGB_TO_DEPTH_PARAMS: RgbToDepthParams; /** * Configuration options for the device camera. */ declare class DeviceCameraOptions { enabled: boolean; /** * Constraints for `getUserMedia`. This will guide the initial camera * selection. */ videoConstraints?: MediaTrackConstraints; /** * Hint for performance optimization on frequent captures. */ willCaptureFrequently: boolean; /** * Parameters for RGB to depth UV mapping given different aspect ratios. */ rgbToDepthParams: RgbToDepthParams; cameraLabel?: string; constructor(options?: DeepReadonly>); } declare const xrDeviceCameraEnvironmentOptions: { readonly enabled: boolean; readonly videoConstraints?: { readonly advanced?: readonly { readonly aspectRatio?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly autoGainControl?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly backgroundBlur?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly channelCount?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly deviceId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly displaySurface?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly echoCancellation?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly facingMode?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly frameRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly groupId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly height?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly noiseSuppression?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly sampleRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly sampleSize?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly width?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; }[] | undefined; readonly aspectRatio?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly autoGainControl?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly backgroundBlur?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly channelCount?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly deviceId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly displaySurface?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly echoCancellation?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly facingMode?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly frameRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly groupId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly height?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly noiseSuppression?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly sampleRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly sampleSize?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly width?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; } | undefined; readonly willCaptureFrequently: boolean; readonly rgbToDepthParams: { readonly scale: number; readonly scaleX: number; readonly scaleY: number; readonly translateU: number; readonly translateV: number; readonly k1: number; readonly k2: number; readonly k3: number; readonly p1: number; readonly p2: number; readonly xc: number; readonly yc: number; }; readonly cameraLabel?: string | undefined; }; declare const xrDeviceCameraUserOptions: { readonly enabled: boolean; readonly videoConstraints?: { readonly advanced?: readonly { readonly aspectRatio?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly autoGainControl?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly backgroundBlur?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly channelCount?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly deviceId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly displaySurface?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly echoCancellation?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly facingMode?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly frameRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly groupId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly height?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly noiseSuppression?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly sampleRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly sampleSize?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly width?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; }[] | undefined; readonly aspectRatio?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly autoGainControl?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly backgroundBlur?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly channelCount?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly deviceId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly displaySurface?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly echoCancellation?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly facingMode?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly frameRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly groupId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly height?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly noiseSuppression?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly sampleRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly sampleSize?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly width?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; } | undefined; readonly willCaptureFrequently: boolean; readonly rgbToDepthParams: { readonly scale: number; readonly scaleX: number; readonly scaleY: number; readonly translateU: number; readonly translateV: number; readonly k1: number; readonly k2: number; readonly k3: number; readonly p1: number; readonly p2: number; readonly xc: number; readonly yc: number; }; readonly cameraLabel?: string | undefined; }; declare const xrDeviceCameraEnvironmentContinuousOptions: { readonly enabled: boolean; readonly videoConstraints?: { readonly advanced?: readonly { readonly aspectRatio?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly autoGainControl?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly backgroundBlur?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly channelCount?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly deviceId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly displaySurface?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly echoCancellation?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly facingMode?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly frameRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly groupId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly height?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly noiseSuppression?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly sampleRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly sampleSize?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly width?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; }[] | undefined; readonly aspectRatio?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly autoGainControl?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly backgroundBlur?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly channelCount?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly deviceId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly displaySurface?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly echoCancellation?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly facingMode?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly frameRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly groupId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly height?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly noiseSuppression?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly sampleRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly sampleSize?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly width?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; } | undefined; readonly willCaptureFrequently: boolean; readonly rgbToDepthParams: { readonly scale: number; readonly scaleX: number; readonly scaleY: number; readonly translateU: number; readonly translateV: number; readonly k1: number; readonly k2: number; readonly k3: number; readonly p1: number; readonly p2: number; readonly xc: number; readonly yc: number; }; readonly cameraLabel?: string | undefined; }; declare const xrDeviceCameraUserContinuousOptions: { readonly enabled: boolean; readonly videoConstraints?: { readonly advanced?: readonly { readonly aspectRatio?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly autoGainControl?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly backgroundBlur?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly channelCount?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly deviceId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly displaySurface?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly echoCancellation?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly facingMode?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly frameRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly groupId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly height?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly noiseSuppression?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly sampleRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly sampleSize?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly width?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; }[] | undefined; readonly aspectRatio?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly autoGainControl?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly backgroundBlur?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly channelCount?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly deviceId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly displaySurface?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly echoCancellation?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly facingMode?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly frameRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly groupId?: string | readonly string[] | { readonly exact?: string | readonly string[] | undefined; readonly ideal?: string | readonly string[] | undefined; } | undefined; readonly height?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly noiseSuppression?: boolean | { readonly exact?: boolean | undefined; readonly ideal?: boolean | undefined; } | undefined; readonly sampleRate?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly sampleSize?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; readonly width?: number | { readonly exact?: number | undefined; readonly ideal?: number | undefined; readonly max?: number | undefined; readonly min?: number | undefined; } | undefined; } | undefined; readonly willCaptureFrequently: boolean; readonly rgbToDepthParams: { readonly scale: number; readonly scaleX: number; readonly scaleY: number; readonly translateU: number; readonly translateV: number; readonly k1: number; readonly k2: number; readonly k3: number; readonly p1: number; readonly p2: number; readonly xc: number; readonly yc: number; }; readonly cameraLabel?: string | undefined; }; declare class SimulatorMediaDeviceInfo { deviceId: string; groupId: string; kind: MediaDeviceKind; label: string; constructor(deviceId?: string, groupId?: string, kind?: MediaDeviceKind, label?: string); } declare class SimulatorCamera { private renderer; private cameraCreated; private cameraInfo?; private mediaStream?; private canvas?; private context?; private fps; matchRenderingCamera: boolean; width: number; height: number; camera: THREE.PerspectiveCamera; constructor(renderer: THREE.WebGLRenderer); init(): void; createSimulatorCamera(): void; enumerateDevices(): Promise; onBeforeSimulatorSceneRender(camera: THREE.Camera, renderScene: (_: THREE.Camera) => void): void; onSimulatorSceneRendered(): void; restartVideoTrack(): void; getMedia(constraints?: MediaTrackConstraints): MediaStream | null | undefined; } /** * Enum for video stream states. */ declare enum StreamState { IDLE = "idle", INITIALIZING = "initializing", STREAMING = "streaming", ERROR = "error", NO_DEVICES_FOUND = "no_devices_found" } type VideoStreamDetails = { force?: boolean; error?: Error; }; interface VideoStreamEventMap extends THREE.Object3DEventMap { statechange: { state: StreamState; details?: T; }; } type VideoStreamGetSnapshotImageDataOptionsBase = { /** The target width, defaults to the video width. */ width?: number; /** The target height, defaults to the video height. */ height?: number; }; type VideoStreamGetSnapshotImageDataOptions = VideoStreamGetSnapshotImageDataOptionsBase & { outputFormat: 'imageData'; }; type VideoStreamGetSnapshotBase64Options = VideoStreamGetSnapshotImageDataOptionsBase & { outputFormat: 'base64'; mimeType?: string; quality?: number; }; type VideoStreamGetSnapshotBlobOptions = VideoStreamGetSnapshotImageDataOptionsBase & { outputFormat: 'blob'; mimeType?: string; quality?: number; }; type VideoStreamGetSnapshotTextureOptions = VideoStreamGetSnapshotImageDataOptionsBase & { outputFormat?: 'texture'; }; type VideoStreamGetSnapshotOptions = VideoStreamGetSnapshotImageDataOptions | VideoStreamGetSnapshotBase64Options | VideoStreamGetSnapshotTextureOptions | VideoStreamGetSnapshotBlobOptions; type VideoStreamOptions = { /** Hint for performance optimization for frequent captures. */ willCaptureFrequently?: boolean; }; /** * The base class for handling video streams (from camera or file), managing * the underlying