import { AxesHelper, DoubleSide, Matrix4, Mesh, MeshBasicMaterial, Object3D, Plane, Raycaster, RingGeometry, Scene, Vector2, Vector3 } from "three"; import { isDevEnvironment, showBalloonWarning } from "../../engine/debug/index.js"; import { AssetReference } from "../../engine/engine_addressables.js"; import { Context } from "../../engine/engine_context.js"; import { destroy, instantiate, isDestroyed } from "../../engine/engine_gameobject.js"; import { InputEventQueue, NEPointerEvent } from "../../engine/engine_input.js"; import { getBoundingBox, getTempVector } from "../../engine/engine_three_utils.js"; import type { IComponent, IGameObject } from "../../engine/engine_types.js"; import { DeviceUtilities, getParam } from "../../engine/engine_utils.js"; import { NeedleXRController, type NeedleXREventArgs, type NeedleXRHitTestResult, NeedleXRSession } from "../../engine/engine_xr.js"; import { Behaviour, GameObject } from "../Component.js"; import type { WebXR } from "./WebXR.js"; // https://github.com/takahirox/takahirox.github.io/blob/master/js.mmdeditor/examples/js/controls/DeviceOrientationControls.js const debug = getParam("debugwebxr"); /** verbose AR touch-transform logging (`?debugartouch`): every tracked touch down/move/up * with raw coordinates and every applied pinch ratio — for diagnosing gesture issues from * device logs */ const debugARTouch = getParam("debugartouch"); const invertForwardMatrix = new Matrix4().makeRotationY(Math.PI); /** a XRRigidTransform pose stored as plain values — XR hit test results and frames are only * valid within the animation frame that produced them, so poses that need to survive until a * later frame (or an input event) must be snapshotted */ type XRAnchorRefPose = { x: number, y: number, z: number, qx: number, qy: number, qz: number, qw: number }; const unitScale = new Vector3(1, 1, 1); const tempAnchorPose = new Matrix4(); const tempAnchorDelta = new Matrix4(); function matrixApproximatelyEquals(a: Matrix4, b: Matrix4, epsilon: number = 1e-6): boolean { for (let i = 0; i < 16; i++) { if (Math.abs(a.elements[i] - b.elements[i]) > epsilon) return false; } return true; } /** * The WebARSessionRoot is the root object for a WebAR session and used to place the scene in AR. * It is also responsible for scaling the user in AR and to define the center of the AR scene. * If not present in the scene it will be created automatically by the WebXR component when entering an AR session. * * **Note**: If the WebXR component {@link WebXR.autoCenter} option is enabled the scene will be automatically centered based on the content in the scene. * * @example Callback when the scene has been placed in AR: * ```ts * WebARSessionRoot.onPlaced((args) => { * console.log("Scene has been placed in AR"); * }); * ``` * * @summary Root object for WebAR sessions, managing scene placement and user scaling in AR. * @category XR * @group Components */ export class WebARSessionRoot extends Behaviour { private static _eventListeners: { [key: string]: Array<(args: { instance: WebARSessionRoot }) => void> } = {}; /** * Event that is called when the scene has been placed in AR. * @param cb the callback that is called when the scene has been placed * @returns a function to remove the event listener */ static onPlaced(cb: (args: { instance: WebARSessionRoot }) => void) { const event = "placed"; if (!this._eventListeners[event]) this._eventListeners[event] = []; this._eventListeners[event].push(cb); return () => { const index = this._eventListeners[event].indexOf(cb); if (index >= 0) this._eventListeners[event].splice(index, 1); } } private static _hasPlaced: boolean = false; /** * @returns true if the scene has been placed in AR by the user or automatic placement */ static get hasPlaced(): boolean { return this._hasPlaced; } /** The scale of the user in AR. * **NOTE**: a large value makes the scene appear smaller * @default 1 */ get arScale(): number { return this._arScale; } set arScale(val: number) { val = Math.max(0.000001, val); // Only rescale the rig on an actual change: onSetScale derives the rig scale from // _arScale, so re-assigning the same value (a sync loop, a UI binding) would wipe // any user two-finger pinch scale (arTouchTransform). if (val === this._arScale) return; this._arScale = val; this.onSetScale(); } private _arScale: number = 1; /** When enabled the placed scene forward direction will towards the XRRig * @deprecated * @default false */ invertForward: boolean = false; /** When assigned this asset will be loaded and visualize the placement while in AR * @default null */ customReticle?: AssetReference; /** Enable touch transform to translate, rotate and scale the scene in AR with multitouch * @default true */ arTouchTransform: boolean = true; /** When enabled the scene will be placed automatically when a point in the real world is found * @default false */ autoPlace: boolean = false; /** When enabled the scene center will be automatically calculated from the content in the scene */ autoCenter: boolean = false; /** When enabled a XR anchor is created at the scene placement position and the scene * stays at that anchored point while tracking refines during the XR session. * On platforms without WebXR anchor support this gracefully falls back to plain placement. * @default true **/ useXRAnchor: boolean = true; /** true if we're currently placing the scene */ private _isPlacing = true; /** This is the world matrix of the ar session root when entering webxr * it is applied when the scene has been placed (e.g. if the session root is x:10, z:10 we want this position to be the center of the scene) */ private readonly _startOffset: Matrix4 = new Matrix4(); private _createdPlacementObject: Object3D | null = null; private readonly _reparentedComponents: Array<{ comp: IComponent, originalObject: IGameObject }> = []; // move objects into a temporary scene while placing (which is not rendered) so that the components won't be disabled during this process // e.g. we want the avatar to still be updated while placing // another possibly solution would be to ensure from this component that the Rig is *also* not disabled while placing private readonly _placementScene: Scene = new Scene(); /** the reticles used for placement */ private readonly _reticle: IGameObject[] = []; /** reference-space hit poses, in sync with the reticles — captured while placing when {@link useXRAnchor} * is enabled, because hit test results can not be used outside the animation frame that produced them */ private readonly _hitRefPoses: (XRAnchorRefPose | undefined)[] = []; private _placementStartTime: number = -1; private _rigPlacementMatrix?: Matrix4; /** if useAnchor is enabled this is the anchor we have created on placing the scene using the placement hit */ private _anchor: XRAnchor | null = null; /** the placement pose (reference space) waiting for anchor creation on the next animation frame */ private _pendingAnchorPose: XRAnchorRefPose | null = null; /** the last anchor pose (in rig space) that has been applied to the rig */ private readonly _anchorLastLocalPose = new Matrix4(); /** user input is used for ar touch transform */ private userInput?: WebXRSessionRootUserInput; onEnable(): void { this.customReticle?.preload(); } supportsXR(mode: XRSessionMode): boolean { return mode === "immersive-ar"; } onEnterXR(_args: NeedleXREventArgs): void { if (debug) console.log("ENTER WEBXR: SessionRoot start..."); this._anchor = null; this._pendingAnchorPose = null; this._hitRefPoses.length = 0; WebARSessionRoot._hasPlaced = false; // if (_args.xr.session.enabledFeatures?.includes("image-tracking")) { // console.warn("Image tracking is enabled - will not place scene"); // return; // } // save the transform of the session root in the scene to apply it when placing the scene this.gameObject.updateMatrixWorld(); this._startOffset.copy(this.gameObject.matrixWorld); // create a new root object for the session placement scripts // and move all the children in the scene in a temporary scene that is not rendered const rootObject = new Object3D(); this._createdPlacementObject = rootObject; rootObject.name = "AR Session Root"; this._placementScene.name = "AR Placement Scene"; this._placementScene.children.length = 0; for (let i = this.context.scene.children.length - 1; i >= 0; i--) { const ch = this.context.scene.children[i]; this._placementScene.add(ch); } this.context.scene.add(rootObject); if (this.autoCenter) { const bounds = getBoundingBox(this._placementScene.children); const center = bounds.getCenter(new Vector3()); const size = bounds.getSize(new Vector3()); const matrix = new Matrix4(); matrix.makeTranslation(center.x, center.y - size.y * .5, center.z); this._startOffset.multiply(matrix); } // reparent components // save which gameobject the sessionroot component was previously attached to this._reparentedComponents.length = 0; this._reparentedComponents.push({ comp: this, originalObject: this.gameObject }); GameObject.addComponent(rootObject, this); // const webXR = GameObject.findObjectOfType(WebXR2); // if (webXR) { // this._reparentedComponents.push({ comp: webXR, originalObject: webXR.gameObject }); // GameObject.addComponent(rootObject, webXR); // const playerSync = GameObject.findObjectOfType(XRFlag); // } // recreate the reticle every time we enter AR for (const ret of this._reticle) { destroy(ret); } this._reticle.length = 0; this._isPlacing = true; // we want to receive pointer events EARLY and prevent interaction with other objects while placing by stopping the event propagation this.context.input.addEventListener("pointerup", this.onPlaceScene, { queue: InputEventQueue.Early }); } onLeaveXR() { // TODO: WebARSessionRoot doesnt work when we enter passthrough and leave XR without having placed the session!!! this.context.input.removeEventListener("pointerup", this.onPlaceScene, { queue: InputEventQueue.Early }); this.onRevertSceneChanges(); // no explicit anchor.delete() — anchors are released by the platform when the session ends this._anchor = null; this._pendingAnchorPose = null; WebARSessionRoot._hasPlaced = false; this._rigPlacementMatrix = undefined; } onUpdateXR(args: NeedleXREventArgs): void { // disable session placement while images are being tracked if (args.xr.isTrackingImages) { for (const ret of this._reticle) ret.visible = false; return; } if (this._isPlacing) { const rigObject = args.xr.rig?.gameObject; // the rig should be parented to the scene while placing // since the camera is always parented to the rig this ensures that the camera is always rendering if (rigObject && rigObject.parent !== this.context.scene) { this.context.scene.add(rigObject); } // in pass through mode we want to place the scene using an XR controller let controllersDidHit = false; // when auto placing we just use the user's view if (args.xr.isPassThrough && args.xr.controllers.length > 0 && !this.autoPlace) { for (const ctrl of args.xr.controllers) { // with this we can only place with the left / first controller right now // we also only have one reticle... this should probably be refactored a bit so we can have multiple reticles // and then place at the reticle for which the user clicked the place button const hit = ctrl.getHitTest(); if (hit) { controllersDidHit = true; this.updateReticleAndHits(args.xr, ctrl.index, hit, args.xr.rigScale); } } } // in screen AR mode we use "camera" hit testing (or when using the simulator where controller hit testing is not supported) if (!controllersDidHit) { const hit = args.xr.getHitTest(); if (hit) { this.updateReticleAndHits(args.xr, 0, hit, args.xr.rigScale); } } } else { // create the anchor (deferred from placement) and keep the scene glued to it if (this.useXRAnchor) this.updateXRAnchor(args.xr); // Scene has been placed if (this.arTouchTransform) { if (!this.userInput) this.userInput = new WebXRSessionRootUserInput(this.context); this.userInput?.enable(); } else this.userInput?.disable(); if (this.arTouchTransform && this.userInput?.hasChanged) { if (args.xr.rig) { const rig = args.xr.rig.gameObject; this.userInput.applyMatrixTo(rig.matrix, true); rig.matrix.decompose(rig.position, rig.quaternion, rig.scale); } this.userInput.reset(); } } } private updateReticleAndHits(xr: NeedleXRSession, i: number, hit: NeedleXRHitTestResult, scale: number) { // capture the reference-space hit pose while the XRFrame that produced the hit is still // active — it can not be queried anymore during placement (which runs from an input event) if (this.useXRAnchor) { const referenceSpace = xr.referenceSpace; const rawPose = referenceSpace ? hit.hit.getPose(referenceSpace) : undefined; if (rawPose) { let pose = this._hitRefPoses[i]; if (!pose) pose = this._hitRefPoses[i] = { x: 0, y: 0, z: 0, qx: 0, qy: 0, qz: 0, qw: 1 }; const { position, orientation } = rawPose.transform; pose.x = position.x; pose.y = position.y; pose.z = position.z; pose.qx = orientation.x; pose.qy = orientation.y; pose.qz = orientation.z; pose.qw = orientation.w; } } let reticle = this._reticle[i]; if (!reticle) { if (this.customReticle) { if (this.customReticle.asset) { reticle = instantiate(this.customReticle.asset); } else { this.customReticle.loadAssetAsync(); return; } } else { reticle = new Mesh( new RingGeometry(0.07, 0.09, 32).rotateX(- Math.PI / 2), new MeshBasicMaterial({ side: DoubleSide, depthTest: false, depthWrite: false, transparent: true, opacity: 1, color: 0xeeeeee }) ) as any as IGameObject; reticle.name = "AR Placement Reticle"; } if (debug) { const axes = new AxesHelper(1); axes.position.y += .01; reticle.add(axes); } this._reticle[i] = reticle; reticle.matrixAutoUpdate = false; reticle.visible = false; } reticle["lastPos"] = reticle["lastPos"] || hit.position.clone(); reticle["lastQuat"] = reticle["lastQuat"] || hit.quaternion.clone(); // reticle["targetPos"] = reticle["targetPos"] || hit.position.clone(); // reticle["targetQuat"] = reticle["targetQuat"] || hit.quaternion.clone(); // TODO we likely want the reticle itself to be placed _exactly_ and then the visuals being lerped, // Right now this leads to a "rotation glitch" when going from a horizontal to a vertical surface reticle.position.copy(reticle["lastPos"].lerp(hit.position, this.context.time.deltaTime / .1)); reticle["lastPos"].copy(reticle.position); reticle.quaternion.copy(reticle["lastQuat"].slerp(hit.quaternion, this.context.time.deltaTime / .05)); reticle["lastQuat"].copy(reticle.quaternion); // TODO make sure original reticle asset scale is respected, or document it should be uniformly scaled // scale *= this.customReticle?.asset?.scale?.x || 1; reticle.scale.set(scale, scale, scale); // if (this.invertForward) { // reticle.rotateY(Math.PI); // } // Workaround: For a custom reticle we apply the view based transform during placement preview // See NE-4161 for context if (this.customReticle) this.applyViewBasedTransform(reticle); reticle.updateMatrix(); reticle.visible = true; if (reticle.parent !== this.context.scene) this.context.scene.add(reticle); if (this._placementStartTime < 0) { this._placementStartTime = this.context.time.realtimeSinceStartup; } if (this.autoPlace) { this.upVec.set(0, 1, 0).applyQuaternion(reticle.quaternion); const isUp = this.upVec.dot(getTempVector(0, 1, 0)) > 0.9; if (isUp) { // We want the reticle to be at a suitable spot for a moment before we place the scene (not place it immediately) let autoplace_timer = reticle["autoplace:timer"] || 0; if (autoplace_timer >= 1) { reticle.visible = false; this.onPlaceScene(null); } else { autoplace_timer += this.context.time.deltaTime; reticle["autoplace:timer"] = autoplace_timer; } } else { reticle["autoplace:timer"] = 0; } } } private onPlaceScene = (evt: NEPointerEvent | null) => { if (this._isPlacing == false) return; if (evt?.used) return; let reticle: IGameObject | undefined = this._reticle[0]; if (!reticle) { console.warn("No reticle to place..."); return; } if (!reticle.visible && !this.autoPlace) { console.warn("Reticle is not visible (can not place)"); return; } if (NeedleXRSession.active?.isTrackingImages) { console.warn("Scene Placement is disabled while images are being tracked"); return; } let hitIndex = 0; if (evt && evt.origin instanceof NeedleXRController) { // until we can use hit testing for both controllers and have multple reticles we only allow placement with the first controller const controllerReticle = this._reticle[evt.origin.index]; if (controllerReticle) { reticle = controllerReticle; hitIndex = evt.origin.index; } } // if we place the scene we don't want this event to be propagated to any sub-objects (via the EventSystem) anymore and trigger e.g. a click on objects for the "place tap" event if (evt) { evt.stopImmediatePropagation(); evt.stopPropagation(); evt.use(); } this._isPlacing = false; this.context.input.removeEventListener("pointerup", this.onPlaceScene); this.onRevertSceneChanges(); // TODO: we should probably use the non-lerped position and quaternion here reticle.position.copy(reticle["lastPos"]); reticle.quaternion.copy(reticle["lastQuat"]); this.onApplyPose(reticle); WebARSessionRoot._hasPlaced = true; if (this.useXRAnchor) { // anchor creation is deferred to the next animation frame — see updateXRAnchor const refPose = this._hitRefPoses[hitIndex]; if (refPose) { this._pendingAnchorPose = { ...refPose }; } else { console.warn("[WebARSessionRoot] can not create a XR anchor: no hit pose was captured for the placement"); if (isDevEnvironment()) showBalloonWarning("Can not create XR anchor: no hit pose available"); } } if (this.context.xr) { for (const ctrl of this.context.xr.controllers) { ctrl.cancelHitTestSource(); } } } private onSetScale() { if (!WebARSessionRoot._hasPlaced) return; const rig = NeedleXRSession.active?.rig?.gameObject; if (rig) { const currentScale = NeedleXRSession.active?.rigScale || 1; const newScale = (1 / this._arScale) * currentScale; const scaleMatrix = new Matrix4().makeScale(newScale, newScale, newScale).invert(); rig.matrix.premultiply(scaleMatrix); rig.matrix.decompose(rig.position, rig.quaternion, rig.scale); } } private onRevertSceneChanges() { for (const ret of this._reticle) { if (!ret) continue; ret.visible = false; ret?.removeFromParent(); } this._reticle.length = 0; for (let i = this._placementScene.children.length - 1; i >= 0; i--) { const ch = this._placementScene.children[i]; this.context.scene.add(ch); } this._createdPlacementObject?.removeFromParent(); for (const reparented of this._reparentedComponents) { // the component (or its original object) may have been destroyed during // teardown — e.g. WebXR destroys the implicit session root it created before // this component's own onLeaveXR runs. A destroyed component must not be // re-added anywhere. if (reparented.comp.destroyed || isDestroyed(reparented.originalObject)) continue; GameObject.addComponent(reparented.originalObject, reparented.comp); } this._reparentedComponents.length = 0; } /** * Deferred anchor creation and per-frame anchor tracking. Called from {@link onUpdateXR} after the scene has been placed. * * Creation can not happen during placement: `XRHitTestResult.createAnchor` only works while the animation * frame that produced the hit is active, and placement runs from an input event where that frame is already * over (`XRFrame.createAnchor` equally requires an active animation frame). So {@link onPlaceScene} only * records the reference-space pose and the anchor is created here, on the next animation frame. * * Once the anchor exists, its pose *changes* (delta in rig space) are applied to the XR rig: the world pose * of the anchored point stays constant while tracking refines, without discarding other rig modifications * ({@link arTouchTransform} user adjustments, {@link arScale}). */ private updateXRAnchor(xr: NeedleXRSession) { const referenceSpace = xr.referenceSpace; if (!referenceSpace) return; const frame = xr.frame; if (this._pendingAnchorPose) { const pending = this._pendingAnchorPose; this._pendingAnchorPose = null; if (typeof frame.createAnchor !== "function" || xr.session.enabledFeatures?.includes("anchors") === false) { // dev-only: useXRAnchor is enabled by default, so platforms without anchor // support (e.g. iOS) would otherwise warn in every production session if (debug || isDevEnvironment()) { console.warn("[WebARSessionRoot] useXRAnchor is enabled but WebXR anchors are not supported by this session"); showBalloonWarning("WebXR anchors are not supported by this session"); } return; } const transform = new XRRigidTransform({ x: pending.x, y: pending.y, z: pending.z }, { x: pending.qx, y: pending.qy, z: pending.qz, w: pending.qw }); // the anchor is created exactly at this pose — remember it (in rig space) as the baseline for deltas const initialPose = xr.convertSpace(transform); this._anchorLastLocalPose.compose(initialPose.position, initialPose.quaternion, unitScale); frame.createAnchor(transform, referenceSpace).then(anchor => { if (xr.running && !this._isPlacing) { this._anchor = anchor; if (debug) console.log("[WebARSessionRoot] created XR anchor", anchor); } else anchor.delete(); }).catch((err: unknown) => { console.warn("[WebARSessionRoot] failed to create XR anchor", err); if (isDevEnvironment()) showBalloonWarning("Failed to create XR anchor"); }); return; } if (!this._anchor) return; const rig = xr.rig?.gameObject; if (!rig) return; const pose = frame.getPose(this._anchor.anchorSpace, referenceSpace); // the anchor may temporarily not be tracked — keep the last applied correction then if (!pose) return; const current = xr.convertSpace(pose.transform); tempAnchorPose.compose(current.position, current.quaternion, unitScale); if (matrixApproximatelyEquals(tempAnchorPose, this._anchorLastLocalPose)) return; // rig_new = rig_old · lastPose · currentPose⁻¹ — keeps the world pose of the anchored // point (rigWorld · anchorRigSpacePose) constant across tracking refinements tempAnchorDelta.copy(tempAnchorPose).invert().premultiply(this._anchorLastLocalPose); this._anchorLastLocalPose.copy(tempAnchorPose); rig.updateMatrix(); rig.matrix.multiply(tempAnchorDelta); rig.matrix.decompose(rig.position, rig.quaternion, rig.scale); if (debug) { const dx = tempAnchorDelta.elements[12], dy = tempAnchorDelta.elements[13], dz = tempAnchorDelta.elements[14]; console.log(`[WebARSessionRoot] applied XR anchor correction: ${(Math.hypot(dx, dy, dz) * 1000).toFixed(2)}mm`); } } private upVec: Vector3 = new Vector3(0, 1, 0); private lookPoint: Vector3 = new Vector3(); private worldUpVec: Vector3 = new Vector3(0, 1, 0); private applyViewBasedTransform(reticle: Object3D) { // Make reticle face the user to unify the placement experience across devices. // The pose that we're receiving from the hit test varies between devices: // - Quest: currently aligned to the mesh that was hit (depends on room setup), has changed a couple times // - Android WebXR: looking at the camera, but pretty random when on a wall // - Mozilla WebXR Viewer: aligned to the start of the session const camGo = this.context.mainCamera as Object3D as GameObject; const reticleGo = reticle as GameObject; const camWP = camGo.worldPosition; const reticleWp = reticleGo.worldPosition; this.upVec.set(0, 1, 0).applyQuaternion(reticle.quaternion); // upVec may be pointing AWAY from us, we have to flip it if that's the case const camPos = camGo.worldPosition; if (camPos) { const camToReticle = reticle.position.clone().sub(camPos); const angle = camToReticle.angleTo(this.upVec); if (angle < Math.PI / 2) { this.upVec.negate(); } } const upAngle = this.upVec.angleTo(this.worldUpVec) * 180 / Math.PI; // For debugging look angle for AR placement // Gizmos.DrawDirection(reticle.position, upVec, "blue", 0.1); // Gizmos.DrawLabel(reticle.position, upAngle.toFixed(2), 0.1); const angleForWallPlacement = 30; if ((upAngle > angleForWallPlacement && upAngle < 180 - angleForWallPlacement) || (upAngle < -angleForWallPlacement && upAngle > -180 + angleForWallPlacement)) { this.lookPoint.copy(reticle.position).add(this.upVec); this.lookPoint.y = reticle.position.y; reticle.lookAt(this.lookPoint); } else { camWP.y = reticleWp.y; reticle.lookAt(camWP); } // TODO: ability to scale the reticle so that we can fit the scene depending on the view angle or distance to the reticle. // Currently, doing this leads to wrong placement of the scene. /* const rigScale = NeedleXRSession.active?.rigScale || 1; const scale = distance * rigScale; reticle.scale.set(scale, scale, scale); */ } private onApplyPose(reticle: Object3D) { const rigObject = NeedleXRSession.active?.rig?.gameObject; if (!rigObject) { console.warn("No rig object to place"); return; } // const rigScale = NeedleXRSession.active?.rigScale || 1; // save the previous rig parent const previousParent = rigObject.parent || this.context.scene; // if we have placed this rig before and this is just "replacing" with the anchor // we need to make sure the XRRig attached to the reticle is at the same position as last time // since in the following code we move it inside the reticle (relative to the reticle) if (this._rigPlacementMatrix) { this._rigPlacementMatrix?.decompose(rigObject.position, rigObject.quaternion, rigObject.scale); } else { this._rigPlacementMatrix = rigObject.matrix.clone(); } this.applyViewBasedTransform(reticle); reticle.updateMatrix(); // attach rig to reticle (since the reticle is in rig space it's a easy way to place the rig where we want it relative to the reticle) this.context.scene.add(reticle); reticle.attach(rigObject); reticle.removeFromParent(); // move rig now relative to the reticle // TODO support scaled reticle rigObject.scale.set(this.arScale, this.arScale, this.arScale); rigObject.position.multiplyScalar(this.arScale); rigObject.updateMatrix(); // if invert forward is disabled we need to invert the forward rotation // we want to look into positive Z direction (if invertForward is enabled we look into negative Z direction) if (this.invertForward) rigObject.matrix.premultiply(invertForwardMatrix); rigObject.matrix.premultiply(this._startOffset); // apply the rig modifications and add it back to the previous parent rigObject.matrix.decompose(rigObject.position, rigObject.quaternion, rigObject.scale); previousParent.add(rigObject); } } class WebXRSessionRootUserInput { private static up = new Vector3(0, 1, 0); private static zero = new Vector3(0, 0, 0); private static one = new Vector3(1, 1, 1); oneFingerDrag: boolean = true; twoFingerRotate: boolean = true; twoFingerScale: boolean = true; readonly context: Context; readonly offset: Matrix4; readonly plane: Plane; private _scale: number = 1; private _hasChanged: boolean = false; get scale() { return this._scale; } // readonly translate: Vector3 = new Vector3(); // readonly rotation: Quaternion = new Quaternion(); // readonly scale: Vector3 = new Vector3(1, 1, 1); constructor(context: Context) { this.context = context; this.offset = new Matrix4() this.plane = new Plane(); this.plane.setFromNormalAndCoplanarPoint(WebXRSessionRootUserInput.up, WebXRSessionRootUserInput.zero); } private _enabled: boolean = false; reset() { this._scale = 1; this.offset.identity(); this._hasChanged = true; } get hasChanged() { return this._hasChanged; } /** * Applies the matrix to the offset matrix * @param matrix the matrix to apply the drag offset to * @param invert if true the offset matrix will be inverted before applying it to the matrix and premultiplied */ applyMatrixTo(matrix: Matrix4, invert: boolean) { this._hasChanged = false; if (invert) { this.offset.invert(); matrix.premultiply(this.offset); } else matrix.multiply(this.offset); // if (this._needsUpdate) // this.updateMatrix(); // matrix.premultiply(this._rotationMatrix); // matrix.premultiply(this.offset).premultiply(this._rotationMatrix) } enable() { if (this._enabled) return; this._enabled = true; // Drive the transform from the engine input system. In AR every touch/select/pinch — // screen tap (transient-pointer), controller or hand — arrives here as a pointer event // with a world-space ray. A pointer that an interactive component (e.g. DragControls) has // consumed is `used` and is ignored below, so touching a draggable object drags only that // object and no longer also slides the whole AR scene. this.context.input.addEventListener("pointerdown", this.onPointerDown); this.context.input.addEventListener("pointermove", this.onPointerMove); this.context.input.addEventListener("pointerup", this.onPointerUp); } disable() { if (!this._enabled) return; this._enabled = false; this.context.input.removeEventListener("pointerdown", this.onPointerDown); this.context.input.removeEventListener("pointermove", this.onPointerMove); this.context.input.removeEventListener("pointerup", this.onPointerUp); this._tracked.clear(); this._hasTwoFingerRef = false; } /** Active, unconsumed pointers and their last screen-pixel position. Gestures are measured in * screen space (device-fixed) rather than world space: the transform moves the rig, which moves * the camera, so world-space ground points would feed back on themselves and jitter/fly away. */ private readonly _tracked = new Map(); private _hasTwoFingerRef = false; private _lastTwoFingerAngle = 0; private _lastTwoFingerDist = 0; // Set whenever the tracked-pointer set changes (finger added/removed). The next move for each // gesture then only RESYNCS the reference point instead of applying a delta — otherwise a // stale `prev` (e.g. the remaining finger after lifting one of a pinch, or a fresh touch whose // first sample already jumped) would translate the scene by a huge amount ("fly away"). private _resyncGesture = false; private isPointerUsed(pointerId: number): boolean { return this.context.input.getIsPointerIdInUse(pointerId); } private onPointerDown = (e: NEPointerEvent) => { // Screen input only for now (handheld AR touch). Controller/hand (spatial) pointers are // rig-relative, so dragging the rig with their world-space ray feeds back on itself — that // needs a rig-independent delta and is a separate follow-up. Screen pointers are // camera-relative and don't have that problem. if (e.mode !== "screen") return; // Only primary presses. if (e.button !== 0) return; // Skip anything a component has grabbed (DragControls etc.) — the whole point of the fix. if (e.used || this.isPointerUsed(e.pointerId)) { if (isDevEnvironment()) console.debug(`[WebARSessionRoot] AR touch #${e.pointerId} ignored for touch transform: pointer is used (grabbed by a component)`); return; } this._tracked.set(e.pointerId, new Vector2(e.clientX, e.clientY)); if (debugARTouch) console.debug(`[ARTouch] down #${e.pointerId} (${e.clientX.toFixed(0)}, ${e.clientY.toFixed(0)}) type=${e.pointerType} tracked=${this._tracked.size}`); else if (isDevEnvironment()) console.debug(`[WebARSessionRoot] AR touch #${e.pointerId} down (tracked fingers: ${this._tracked.size})`); // the pointer count changed — resync references on the next move before applying anything this._hasTwoFingerRef = false; this._resyncGesture = true; }; private onPointerMove = (e: NEPointerEvent) => { // pointerIds are only unique per device family: XR controller/transient-pointer events // reuse ids 0/1 and carry no meaningful clientX/Y — without this gate a controller move // teleports a tracked finger to (0,0) and corrupts the pinch distance/angle if (e.mode !== "screen") return; const prev = this._tracked.get(e.pointerId); if (!prev) return; // A pointer can be consumed after it went down (e.g. drag threshold reached) — drop it. if (this.isPointerUsed(e.pointerId)) { if (isDevEnvironment()) console.debug(`[WebARSessionRoot] AR touch #${e.pointerId} dropped from touch transform: pointer became used mid-gesture`); this._tracked.delete(e.pointerId); this._hasTwoFingerRef = false; this._resyncGesture = true; return; } const curX = e.clientX, curY = e.clientY; // After a finger was added/removed, the first move only resyncs the reference (no delta), // so we never translate/scale by a stale gap. if (this._resyncGesture) { prev.set(curX, curY); this._resyncGesture = false; this._hasTwoFingerRef = false; return; } if (this._tracked.size === 1) { // Translate: unproject the PREVIOUS and CURRENT pixel through the SAME (current) camera // onto the ground plane. Using one camera for both makes the delta the finger's actual // movement, with none of the rig-movement feedback that world-point tracking suffered. if (this.oneFingerDrag && this.pixelToGround(curX, curY, this._curW) && this.pixelToGround(prev.x, prev.y, this._prevW)) { this.addMovement(this._curW.x - this._prevW.x, this._curW.z - this._prevW.z); } prev.set(curX, curY); } else if (this._tracked.size === 2) { prev.set(curX, curY); const it = this._tracked.values(); const a = it.next().value as Vector2; const b = it.next().value as Vector2; // Pinch distance and angle in SCREEN pixels — independent of the rig/camera, so no feedback. const angle = Math.atan2(b.y - a.y, b.x - a.x); const dist = a.distanceTo(b); if (debugARTouch) console.debug(`[ARTouch] move #${e.pointerId} (${curX.toFixed(0)}, ${curY.toFixed(0)}) a=(${a.x.toFixed(0)}, ${a.y.toFixed(0)}) b=(${b.x.toFixed(0)}, ${b.y.toFixed(0)}) dist=${dist.toFixed(1)} last=${this._lastTwoFingerDist.toFixed(1)} ref=${this._hasTwoFingerRef}`); if (this._hasTwoFingerRef) { if (this.twoFingerRotate) { const dAngle = angle - this._lastTwoFingerAngle; if (Math.abs(dAngle) > 0.001) this.addRotation(dAngle); } if (this.twoFingerScale && this._lastTwoFingerDist > 1 && dist > 1) { // Pivot at the ground point under the pinch midpoint (fallback: the ground // below the camera when the midpoint ray misses/grazes the plane). let pivotX: number, pivotZ: number; if (this.pixelToGround((a.x + b.x) / 2, (a.y + b.y) / 2, this._pivotW)) { pivotX = this._pivotW.x; pivotZ = this._pivotW.z; } else { const cam = this.context.mainCamera; pivotX = cam?.worldPosition.x ?? 0; pivotZ = cam?.worldPosition.z ?? 0; } this.applyScaleRatio(dist / this._lastTwoFingerDist, pivotX, pivotZ); } } this._lastTwoFingerAngle = angle; this._lastTwoFingerDist = dist; this._hasTwoFingerRef = true; } }; private onPointerUp = (e: NEPointerEvent) => { // see onPointerMove — a controller up with a colliding id must not end a touch gesture if (e.mode !== "screen") return; if (debugARTouch) console.debug(`[ARTouch] up #${e.pointerId} (${e.clientX.toFixed(0)}, ${e.clientY.toFixed(0)}) tracked=${this._tracked.size - (this._tracked.has(e.pointerId) ? 1 : 0)}`); this._tracked.delete(e.pointerId); this._hasTwoFingerRef = false; // remaining pointers must resync before applying (avoids a jump when lifting one of two) this._resyncGesture = true; if (this._tracked.size < 2 && this._gestureScale !== 1) { if (isDevEnvironment()) console.debug(`[WebARSessionRoot] pinch scale gesture ended: total ×${this._gestureScale.toFixed(3)}, rig scale now ${(NeedleXRSession.active?.rigScale ?? 1).toFixed(3)}`); this._gestureScale = 1; } }; // private _needsUpdate: boolean = true; // private _rotationMatrix: Matrix4 = new Matrix4(); // private updateMatrix() { // this._needsUpdate = false; // this._rotationMatrix.makeRotationFromQuaternion(this.rotation); // this.offset.compose(this.translate, new Quaternion(), this.scale); // // const rot = this._tempMatrix.makeRotationY(this.angle); // // this.translate.applyMatrix4(rot); // // this.offset.elements[12] = this.translate.x; // // this.offset.elements[13] = this.translate.y; // // this.offset.elements[14] = this.translate.z; // // this.offset.premultiply(rot); // // const s = this.scale; // // this.offset.premultiply(this._tempMatrix.makeScale(s, s, s)); // } private readonly _raycaster: Raycaster = new Raycaster(); private readonly _screenPos: Vector3 = new Vector3(); private readonly _camPos: Vector3 = new Vector3(); private readonly _curW: Vector3 = new Vector3(); private readonly _prevW: Vector3 = new Vector3(); private readonly _pivotW: Vector3 = new Vector3(); /** cumulative scale ratio of the current pinch gesture — dev logging only */ private _gestureScale: number = 1; /** Unproject a screen pixel through the current main camera onto the ground plane (world space), * writing into `target`. Returns false when the ray is parallel to / points away from the plane. */ private pixelToGround(px: number, py: number, target: Vector3): boolean { const camera = this.context.mainCamera; if (!camera) return false; this._screenPos.set((px / window.innerWidth) * 2 - 1, -(py / window.innerHeight) * 2 + 1, 1); this._screenPos.unproject(camera); this._camPos.copy(camera.worldPosition); const dir = this._screenPos.sub(this._camPos).normalize(); // Reject rays that graze the ground plane (near the horizon): the intersection would be // effectively at infinity, so a tiny finger move would translate the scene enormously. if (Math.abs(dir.y) < 0.08) return false; this._raycaster.set(this._camPos, dir); return this._raycaster.ray.intersectPlane(this.plane, target) !== null; } private addMovement(dx: number, dz: number) { // Translate the rig by the raw world-space ground-plane delta so the grabbed point tracks // the finger 1:1. The `_scale` term keeps that consistent while a two-finger pinch is also // scaling. Do NOT multiply by the rig scale (the old `* factor`): the ground delta is // already in world space, and because the camera follows the rig, over-translating // compounds through the feedback loop into an exponential runaway ("scene flies away") on // scaled-up AR scenes. dx /= this._scale; dz /= this._scale; // apply it this.offset.elements[12] += dx; this.offset.elements[14] += dz; if (dx !== 0 || dz !== 0) this._hasChanged = true; }; private readonly _tempMatrix: Matrix4 = new Matrix4(); /** Scale the AR scene by a pinch ratio (current two-pointer screen distance / previous), * about the world-space ground point (pivotX, 0, pivotZ). */ private applyScaleRatio(ratio: number, pivotX: number, pivotZ: number) { if (!isFinite(ratio) || ratio <= 0 || ratio === 1) return; // The offset is applied INVERTED to the rig, so scaling the offset by `ratio` scales the // rig by 1/ratio — i.e. pinch OUT (ratio > 1) makes the user smaller relative to the scene, // so the scene appears bigger. _scale tracks the resulting scene scale (reciprocal of the // rig scale) and keeps the one-finger drag speed consistent. this._scale *= 1 / ratio; this._gestureScale *= ratio; // Scale about the pinch pivot — T(p)·S(r)·T(-p), i.e. diagonal r with translation // p·(1-r) — so the spot under the fingers stays fixed while the scene grows around // it. Scaling about the world origin instead is dominated by its translation term // when the user stands inside a large scene far from the scene origin: everything // slides by (r-1)·|origin distance| while barely appearing to grow ("scale has no // effect", observed with a room-scale splat scan). Pivot y stays 0: scaling about a // ground-plane point keeps the scene on the ground. const m = this._tempMatrix.makeScale(ratio, ratio, ratio); m.elements[12] = pivotX * (1 - ratio); m.elements[14] = pivotZ * (1 - ratio); this.offset.premultiply(m); this._hasChanged = true; } private addRotation(rot: number) { rot *= -1; // this.rotation.multiply(new Quaternion().setFromAxisAngle(WebXRSessionRootUserInput.up, rot)); // this._needsUpdate = true; // return; this._tempMatrix.makeRotationY(rot); this.offset.premultiply(this._tempMatrix); if (rot !== 0) this._hasChanged = true; } }