import * as THREE from 'three'; import {XRControllerModelFactory} from 'three/addons/webxr/XRControllerModelFactory.js'; import {XRHandModelFactory} from 'three/addons/webxr/XRHandModelFactory.js'; import {NUM_HANDS} from '../constants'; import {Options} from '../core/Options.js'; import type {SupportedRenderer} from '../core/RendererTypes'; import {KeyEvent} from '../core/Script'; import type { DirectTouchInput, InteractionFrameInput, RaySourceInput, } from '../interaction/InteractionTypes.js'; import {Reticle} from '../interaction/reticle/Reticle.js'; import {ControllerRayVisual} from './components/ControllerRayVisual'; import type { Controller, ControllerEvent, ControllerEventMap, } from './Controller'; import {GamepadController} from './GamepadController'; import {GazeController} from './GazeController'; import {HeadGestureRecognition} from './headGestures/HeadGestureRecognition'; import {MouseController} from './MouseController'; import {XRSystems} from '../core/components/XRSystems'; import {PinchFilter} from './PinchFilter'; export class ActiveControllers extends THREE.Group { type = 'ActiveControllers'; name = 'Active Controllers'; } export class Reticles extends THREE.Group { type = 'Reticles'; name = 'Reticles'; } interface TransientPointerGraceFrame { position: THREE.Vector3; orientation: THREE.Quaternion; sourceType: RaySourceInput['sourceType']; sampled: boolean; intersection?: THREE.Intersection; } /** * Holds physical input sources and samples their current state each frame. */ export class Input { options!: Options; controllers: Controller[] = []; controllerGrips: THREE.Group[] = []; hands: THREE.XRHandSpace[] = []; /** Completed head gestures, when enabled before initialization. */ headGestures?: HeadGestureRecognition; pivotsEnabled = false; gazeController = new GazeController(); mouseController = new MouseController(); gamepadController = new GamepadController(); private readonly builtInControllers = new Set([ this.gazeController, this.mouseController, this.gamepadController, ]); private readonly addonControllers = new Set(); private readonly removingControllers = new Map>(); private nextControllerId = 0; controllersEnabled = true; listeners = new Map(); private dispatchControllerEvent = (event: ControllerEvent) => this.dispatchEvent(event); private pinchFilter = new PinchFilter(this.dispatchControllerEvent); private releasedControllers = new Set(); private nativeSelectControllers = new Set(); private disconnectedTransientPointerControllers = new Map< Controller, TransientPointerGraceFrame >(); private keyDownListeners = new Set<(event: KeyEvent) => void>(); private keyUpListeners = new Set<(event: KeyEvent) => void>(); activeControllers = new ActiveControllers(); leftController?: Controller; rightController?: Controller; reticles = new Reticles(); private ownedReticles = new Set( this.gazeController.reticle ? [this.gazeController.reticle] : [] ); private reticleConfigurer?: (reticle: Reticle) => void; private readonly raySourceInputs: RaySourceInput[] = []; private readonly raySourceSlots = new Map(); private readonly directTouchInputs: DirectTouchInput[] = []; private readonly directTouchSlots: Array = []; private readonly interactionFrame: InteractionFrameInput = { raySources: this.raySourceInputs, directTouches: this.directTouchInputs, }; /** * Initializes physical input sources. Only called by Core. */ init({ systemsGroup, options, renderer, }: { systemsGroup: XRSystems; options: Options; renderer: SupportedRenderer; }) { systemsGroup.add(this.activeControllers, this.reticles); this.controllersEnabled = options.controllers.enabled; this.options = options; if (options.headGestures.enabled) { this.headGestures = new HeadGestureRecognition(); systemsGroup.add(this.headGestures); } if (!options.controllers.enabled) { return; } const controllers = this.controllers; const controllerGrips = this.controllerGrips; for (let i = 0; i < NUM_HANDS; ++i) { controllers.push(renderer.xr.getController(i)); controllers[i].userData.id = i; this.builtInControllers.add(controllers[i]); this.activeControllers.add(this.controllers[i]); } controllers.push(this.gazeController); controllers.push(this.mouseController); this.activeControllers.add(this.mouseController); controllers.push(this.gamepadController); this.activeControllers.add(this.gamepadController); if (options.controllers.enabled) { if (options.controllers.visualization) { const controllerModelFactory = new XRControllerModelFactory(); for (let i = 0; i < NUM_HANDS; ++i) { controllerGrips.push(renderer.xr.getControllerGrip(i)); controllerGrips[i].add( controllerModelFactory.createControllerModel(controllerGrips[i]) ); this.activeControllers.add(controllerGrips[i]); } } // TODO: Separate logic to XR Hands. if (options.hands.enabled) { for (let i = 0; i < NUM_HANDS; ++i) { this.hands.push(renderer.xr.getHand(i)); this.activeControllers.add(this.hands[i]); } if (options.hands.visualization) { if (options.hands.visualizeJoints) { console.log('Visualize hand joints.'); const handModelFactory = new XRHandModelFactory(); for (let i = 0; i < NUM_HANDS; ++i) { const handModel = handModelFactory.createHandModel( this.hands[i], 'boxes' ); handModel.xb = {...handModel.xb, pointerEvents: 'none'}; this.hands[i].add(handModel); } } if (options.hands.visualizeMeshes) { console.log('Visualize hand meshes.'); const handModelFactory = new XRHandModelFactory(); for (let i = 0; i < NUM_HANDS; ++i) { const handModel = handModelFactory.createHandModel( this.hands[i], 'mesh' ); handModel.xb = {...handModel.xb, pointerEvents: 'none'}; this.hands[i].add(handModel); } } } } } if (options.controllers.visualizeRays) { for (let i = 0; i < NUM_HANDS; ++i) { controllers[i].add(new ControllerRayVisual()); } } for (const controller of controllers) { this.setupNativeSelectTracking(controller); } this.bindSelectStart(this.defaultOnSelectStart.bind(this)); this.bindSelectEnd(this.defaultOnSelectEnd.bind(this)); this.bindSqueezeStart(this.defaultOnSqueezeStart.bind(this)); this.bindSqueezeEnd(this.defaultOnSqueezeEnd.bind(this)); this.bindListener('connected', this.defaultOnConnected.bind(this)); this.bindListener('disconnected', this.defaultOnDisconnected.bind(this)); } /** * Retrieves the controller object by its ID. * @param id - The ID of the controller. * @returns The controller with the specified ID, or undefined after removal. */ get(id: number): Controller | undefined { return this.controllers.find((controller) => controller.userData.id === id); } /** * Adds an object to both controllers by creating a new group and cloning it. * @param obj - The object to add to each controller. */ addObject(obj: THREE.Object3D) { const group = new THREE.Group(); group.add(obj); // Clones the group for each controller, adding it to the controller. for (let i = 0; i < this.controllers.length; ++i) { this.controllers[i].add(group.clone()); } } /** * Creates a pivot point for each hand, primarily used as a reference * point. */ enablePivots() { if (this.pivotsEnabled) return; this.pivotsEnabled = true; const pivot = new THREE.Mesh(new THREE.IcosahedronGeometry(0.01, 3)); pivot.name = 'pivot'; pivot.position.z = -0.05; this.addObject(pivot); } /** * Adds reticles to the controllers and scene, with initial visibility set to * false. */ addReticles() { let id = 0; for (const controller of this.controllers) { if (controller.reticle == null) { controller.reticle = new Reticle(); this.ownedReticles.add(controller.reticle); controller.reticle.name = 'Reticle ' + id; ++id; } controller.reticle.visible = false; this.reticles.add(controller.reticle); if (this.reticleConfigurer && controller.reticle) { this.reticleConfigurer(controller.reticle); } } } /** * Sets a configuration callback for reticles (such as upgrading to WebGPU materials) * and immediately applies it to all existing reticles. */ setReticleConfigurer(configurer: (reticle: Reticle) => void) { this.reticleConfigurer = configurer; const configured = new Set(); for (const reticle of this.ownedReticles) { configurer(reticle); configured.add(reticle); } for (const controller of this.controllers) { if (controller.reticle && !configured.has(controller.reticle)) { configurer(controller.reticle); configured.add(controller.reticle); } } } /** * Default action to handle the start of a selection, setting the selecting * state to true. */ defaultOnSelectStart(event: ControllerEvent) { const controller = event.target; controller.userData.selected = true; } /** * Default action to handle the end of a selection, setting the selecting * state to false. */ defaultOnSelectEnd(event: ControllerEvent) { const controller = event.target; controller.userData.selected = false; this.releasedControllers.add(controller); } defaultOnSqueezeStart(event: ControllerEvent) { const controller = event.target; controller.userData.squeezing = true; this.defaultOnSelectStart(event); } defaultOnSqueezeEnd(event: ControllerEvent) { const controller = event.target; controller.userData.squeezing = false; this.defaultOnSelectEnd(event); } defaultOnConnected(event: ControllerEvent) { const controller = event.target; controller.userData.connected = true; controller.gamepad = event.data?.gamepad; controller.inputSource = event.data; switch (event.data?.handedness) { case 'left': this.leftController = controller; break; case 'right': this.rightController = controller; break; } } defaultOnDisconnected(event: ControllerEvent) { const controller = event.target; const wasSelected = controller.userData.selected === true; const nativeSelectArrived = this.nativeSelectControllers.delete(controller); const shouldKeepTransientPointer = nativeSelectArrived && (wasSelected || this.releasedControllers.has(controller)) && (event.data?.targetRayMode ?? controller.inputSource?.targetRayMode) === 'transient-pointer'; if (shouldKeepTransientPointer) { const lastIntersection = controller.userData.lastIntersection as | THREE.Intersection | null | undefined; this.disconnectedTransientPointerControllers.set(controller, { position: controller.getWorldPosition(new THREE.Vector3()), orientation: controller.getWorldQuaternion(new THREE.Quaternion()), sourceType: this.getRaySourceType(controller), sampled: false, ...(lastIntersection?.object && lastIntersection.point ? { intersection: { ...lastIntersection, point: lastIntersection.point.clone(), ...(lastIntersection.uv ? {uv: lastIntersection.uv.clone()} : {}), ...(lastIntersection.uv1 ? {uv1: lastIntersection.uv1.clone()} : {}), ...(lastIntersection.normal ? {normal: lastIntersection.normal.clone()} : {}), ...(lastIntersection.barycoord ? {barycoord: lastIntersection.barycoord.clone()} : {}), ...(lastIntersection.pointOnLine ? {pointOnLine: lastIntersection.pointOnLine.clone()} : {}), }, } : {}), }); } controller.userData.connected = false; if (controller.userData.selected) { controller.userData.selected = false; this.dispatchEvent({ type: 'selectend', target: controller, data: event.data, isCustom: true, } as unknown as ControllerEvent); } if (controller.reticle) { controller.reticle.visible = false; } if (!shouldKeepTransientPointer) { this.releasedControllers.delete(controller); } delete controller?.gamepad; if (this.leftController === controller) this.leftController = undefined; if (this.rightController === controller) this.rightController = undefined; } private captureNativeSelectEvent = (event: ControllerEvent) => { if (event.type === 'selectstart') { this.nativeSelectControllers.delete(event.target); } else if ( event.type === 'select' && !(event as ControllerEvent & {isCustom?: boolean}).isCustom ) { this.nativeSelectControllers.add(event.target); } }; private setupNativeSelectTracking(controller: Controller) { controller.addEventListener('selectstart', this.captureNativeSelectEvent); controller.addEventListener('select', this.captureNativeSelectEvent); } private removeNativeSelectTracking(controller: Controller) { controller.removeEventListener( 'selectstart', this.captureNativeSelectEvent ); controller.removeEventListener('select', this.captureNativeSelectEvent); } /** * Binds a listener to both controllers. * @param listenerName - Event name * @param listener - Function to call */ bindListener( listenerName: keyof ControllerEventMap, listener: (event: ControllerEvent) => void ) { for (const controller of this.controllers) { this.setupNativeSelectTracking(controller); this.pinchFilter.setupControllerForType(controller, listenerName); } if (!this.listeners.has(listenerName)) { this.listeners.set(listenerName, []); } this.listeners.get(listenerName).push(listener); } unbindListener( listenerName: keyof ControllerEventMap, listener: (event: ControllerEvent) => void ) { if (this.listeners.has(listenerName)) { const list = this.listeners.get(listenerName)!; const index = list.indexOf(listener); if (index !== -1) { list.splice(index, 1); } if (list.length === 0) { for (const controller of this.controllers) { this.pinchFilter.removeControllerForType(controller, listenerName); } } } } dispatchEvent(event: ControllerEvent) { const removal = this.removingControllers.get(event.target); const terminal = removal && (event.type === 'selectend' || event.type === 'disconnected'); if (terminal) { // A disconnect hook may emit these itself, and defaultOnDisconnected may // synthesize selectend. Input observers still receive each exactly once. if (removal.has(event.type)) return; removal.add(event.type); } else if (this.pinchFilter.shouldFilterEvent(event)) { return; } if (this.listeners.has(event.type)) { for (const listener of this.listeners.get(event.type)) { listener(event); } } } /** * Binds an event listener to handle 'selectstart' events for both * controllers. * @param event - The event listener function. */ bindSelectStart(event: (event: ControllerEvent) => void) { this.bindListener('selectstart', event); } /** * Binds an event listener to handle 'selectend' events for both controllers. * @param event - The event listener function. */ bindSelectEnd(event: (event: ControllerEvent) => void) { this.bindListener('selectend', event); } /** * Binds an event listener to handle 'select' events for both controllers. * @param event - The event listener function. */ bindSelect(event: (event: ControllerEvent) => void) { this.bindListener('select', event); } /** * Binds an event listener to handle 'squeezestart' events for both * controllers. * @param event - The event listener function. */ bindSqueezeStart(event: (event: ControllerEvent) => void) { this.bindListener('squeezestart', event); } /** * Binds an event listener to handle 'squeezeend' events for both controllers. * @param event - The event listener function. */ bindSqueezeEnd(event: (event: ControllerEvent) => void) { this.bindListener('squeezeend', event); } bindSqueeze(event: (event: ControllerEvent) => void) { this.bindListener('squeeze', event); } bindKeyDown(event: (event: KeyEvent) => void) { if (this.keyDownListeners.has(event)) return; this.keyDownListeners.add(event); if (typeof window !== 'undefined') { window.addEventListener('keydown', event); } } bindKeyUp(event: (event: KeyEvent) => void) { if (this.keyUpListeners.has(event)) return; this.keyUpListeners.add(event); if (typeof window !== 'undefined') { window.addEventListener('keyup', event); } } unbindKeyDown(event: (event: KeyEvent) => void) { this.keyDownListeners.delete(event); if (typeof window !== 'undefined') { window.removeEventListener('keydown', event); } } unbindKeyUp(event: (event: KeyEvent) => void) { this.keyUpListeners.delete(event); if (typeof window !== 'undefined') { window.removeEventListener('keyup', event); } } /** Samples current controller, button, and direct-touch state. */ sampleSources() { for (const [controller, graceFrame] of this .disconnectedTransientPointerControllers) { if (graceFrame.sampled) { this.disconnectedTransientPointerControllers.delete(controller); this.releasedControllers.delete(controller); } else { graceFrame.sampled = true; } } if (this.controllersEnabled) { for (const controller of this.controllers) { if (controller.userData.connected !== true) continue; controller.updatePose?.(); this.pinchFilter.updateController(controller); } } this.updateDirectTouchInputs(); } /** Returns the complete physical source state sampled this frame. */ getFrame(): InteractionFrameInput { this.raySourceInputs.length = 0; if (this.controllersEnabled) { for (const controller of this.controllers) { const transientPointerGraceFrame = this.disconnectedTransientPointerControllers.get(controller); if ( controller.userData.connected !== true && !transientPointerGraceFrame ) { continue; } let input = this.raySourceSlots.get(controller); if (!input) { input = { controller, sourceType: this.getRaySourceType(controller), ray: new THREE.Ray(), selected: false, position: new THREE.Vector3(), orientation: new THREE.Quaternion(), }; this.raySourceSlots.set(controller, input); } const position = input.position!; const orientation = input.orientation!; if (transientPointerGraceFrame) { position.copy(transientPointerGraceFrame.position); orientation.copy(transientPointerGraceFrame.orientation); } else { controller.getWorldPosition(position); controller.getWorldQuaternion(orientation); } input.sourceType = transientPointerGraceFrame?.sourceType ?? this.getRaySourceType(controller); input.ray.origin.copy(position); input.ray.direction .set(0, 0, -1) .applyQuaternion(orientation) .normalize(); if (transientPointerGraceFrame) { input.intersections = transientPointerGraceFrame.intersection ? [transientPointerGraceFrame.intersection] : undefined; } else { const lastIntersection = controller.userData.lastIntersection as | THREE.Intersection | null | undefined; input.intersections = lastIntersection && lastIntersection.object && lastIntersection.point ? [lastIntersection] : undefined; } input.selected = !transientPointerGraceFrame && controller.userData.selected === true; input.released = transientPointerGraceFrame ? this.releasedControllers.has(controller) : this.releasedControllers.delete(controller); this.raySourceInputs.push(input); } } return this.interactionFrame; } private getRaySourceType( controller: Controller ): RaySourceInput['sourceType'] { if (controller === this.mouseController) return 'mouse'; if (controller === this.gazeController) return 'gaze'; if (controller.inputSource?.hand) return 'hand-ray'; return 'controller-ray'; } private updateDirectTouchInputs() { this.directTouchInputs.length = 0; for (let handIndex = 0; handIndex < NUM_HANDS; handIndex++) { const controller = this.controllers[handIndex]; const hand = this.hands[handIndex]; const indexTip = hand?.joints?.['index-finger-tip']; // Three.js retains joint poses after tracking is lost. A stale touch // would keep suppressing the controller's ray and reticle. if (!controller || !hand?.visible || !indexTip?.visible) continue; let input = this.directTouchSlots[handIndex]; if (!input) { input = { controller, handIndex, point: new THREE.Vector3(), orientation: new THREE.Quaternion(), selected: false, }; this.directTouchSlots[handIndex] = input; } input.controller = controller; input.hand = hand.joints?.wrist; indexTip.getWorldPosition(input.point); controller.getWorldQuaternion(input.orientation!); input.selected = controller.userData.selected === true; this.directTouchInputs.push(input); } } enableGazeController() { this.activeControllers.add(this.gazeController); this.gazeController.connect(); } disableGazeController() { this.gazeController.disconnect(); this.activeControllers.remove(this.gazeController); } private registerController(controller: Controller) { if (this.controllers.includes(controller)) return; // Native slots/built-ins may grow independently (e.g. transient XR slots). // Never recycle an addon's ID when a previous source is removed. for (const existing of [...this.builtInControllers, ...this.controllers]) { const id = existing.userData.id; if (Number.isSafeInteger(id) && id >= 0) { this.nextControllerId = Math.max(this.nextControllerId, id + 1); } } controller.userData.id = this.nextControllerId++; this.controllers.push(controller); if (!this.builtInControllers.has(controller)) this.addonControllers.add(controller); if (!controller.reticle && this.options?.reticles.enabled) { controller.reticle = new Reticle(); this.ownedReticles.add(controller.reticle); } if (controller.reticle) { controller.reticle.visible = false; this.reticles.add(controller.reticle); if (this.reticleConfigurer) { this.reticleConfigurer(controller.reticle); } } this.setupNativeSelectTracking(controller); this.pinchFilter.setupController(controller, this.listeners.keys()); } enableController(controller: Controller) { if (this.removingControllers.has(controller)) return; this.registerController(controller); this.activeControllers.add(controller as unknown as THREE.Object3D); (controller as unknown as {connect?: () => void}).connect?.(); } disableController(controller: Controller) { (controller as unknown as {disconnect?: () => void}).disconnect?.(); this.activeControllers.remove(controller as unknown as THREE.Object3D); } /** * Permanently unregisters a source added through enableController. Built-ins * and already-removed sources are ignored. Capture is cancelled as source-lost * next frame; terminal selectend/disconnected notifications are sent once. */ removeController(controller: Controller) { if ( this.builtInControllers.has(controller) || !this.addonControllers.delete(controller) ) return; const index = this.controllers.indexOf(controller); if (index !== -1) this.controllers.splice(index, 1); this.removeNativeSelectTracking(controller); this.nativeSelectControllers.delete(controller); this.disconnectedTransientPointerControllers.delete(controller); this.removingControllers.set(controller, new Set()); const seen = new Set(); const observeEnd = () => { seen.add('selectend'); }; const observeDisconnect = () => { seen.add('disconnected'); }; controller.addEventListener('selectend', observeEnd); controller.addEventListener('disconnected', observeDisconnect); this.pinchFilter.setupController(controller, ['selectend', 'disconnected']); try { (controller as unknown as {disconnect?: () => void}).disconnect?.(); } finally { try { try { if (!seen.has('selectend')) controller.dispatchEvent({type: 'selectend', target: controller}); } finally { if (!seen.has('disconnected')) controller.dispatchEvent({ type: 'disconnected', target: controller, }); } } finally { controller.removeEventListener('selectend', observeEnd); controller.removeEventListener('disconnected', observeDisconnect); for (const type of new Set([ ...this.listeners.keys(), 'selectend', 'disconnected', ])) { this.pinchFilter.removeControllerForType(controller, type); } this.removingControllers.delete(controller); this.activeControllers.remove(controller); controller.userData.connected = false; controller.userData.selected = false; controller.userData.squeezing = false; delete controller.gamepad; if (this.leftController === controller) this.leftController = undefined; if (this.rightController === controller) this.rightController = undefined; this.releasedControllers.delete(controller); this.nativeSelectControllers.delete(controller); this.disconnectedTransientPointerControllers.delete(controller); this.raySourceSlots.delete(controller); if (controller.reticle) { controller.reticle.visible = false; controller.reticle.targetObject = undefined; controller.reticle.removeFromParent(); if (this.ownedReticles.delete(controller.reticle)) { controller.reticle.dispose(); controller.reticle = undefined; } } } } } disableControllers() { this.controllersEnabled = false; this.nativeSelectControllers.clear(); this.disconnectedTransientPointerControllers.clear(); for (const controller of this.controllers) { controller.userData.selected = false; this.releasedControllers.delete(controller); if (controller.reticle) { controller.reticle.visible = false; controller.reticle.targetObject = undefined; } } } enableControllers() { this.controllersEnabled = true; } dispose() { if (typeof window !== 'undefined') { for (const listener of this.keyDownListeners) { window.removeEventListener('keydown', listener); } for (const listener of this.keyUpListeners) { window.removeEventListener('keyup', listener); } } this.keyDownListeners.clear(); this.keyUpListeners.clear(); this.pinchFilter.dispose(this.controllers); this.addonControllers.clear(); this.removingControllers.clear(); for (const controller of this.controllers) { this.removeNativeSelectTracking(controller); } this.listeners.clear(); for (const controller of new Set([ ...this.controllers, this.gazeController, ])) { if (controller.reticle && this.ownedReticles.has(controller.reticle)) { controller.reticle = undefined; } } for (const reticle of this.ownedReticles) reticle.dispose(); this.ownedReticles.clear(); this.reticles.clear(); this.raySourceInputs.length = 0; this.raySourceSlots.clear(); this.nativeSelectControllers.clear(); this.disconnectedTransientPointerControllers.clear(); this.directTouchInputs.length = 0; this.directTouchSlots.length = 0; } }