import { Camera, Object3D, PerspectiveCamera, Quaternion, Vector3 } from "three"; import { enableSpatialConsole, isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../debug/index.js"; import { Application, internalOnUserInputRegistered } from "../engine_application.js"; import { Context, FrameEvent } from "../engine_context.js"; import { ContextEvent, ContextRegistry } from "../engine_context_registry.js"; import { isDestroyed } from "../engine_gameobject.js"; import { Gizmos } from "../engine_gizmos.js"; import { Telemetry } from "../engine_license.js"; import { registerFrameEventCallback, unregisterFrameEventCallback } from "../engine_lifecycle_functions_internal.js"; import { getBoundingBox, getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../engine_three_utils.js"; import type { ICamera, IComponent, INeedleXRSession } from "../engine_types.js"; import { delay, DeviceUtilities, getParam } from "../engine_utils.js"; import { invokeXRSessionEnd, invokeXRSessionStart } from "./events.js" import { flipForwardMatrix, flipForwardQuaternion, ImplictXRRig } from "./internal.js"; import { NeedleXRController } from "./NeedleXRController.js"; import { NeedleXRSync } from "./NeedleXRSync.js"; import { SceneTransition } from "./SceneTransition.js"; import { SessionInfo, TemporaryXRContext } from "./TempXRContext.js"; import { InternalUSDZRegistry } from "./usdz.js"; import type { IXRRig } from "./XRRig.js"; const measure_SessionStartedMarker = "NeedleXRSession onStart"; const measure_SessionEndedMarker = "NeedleXRSession onEnd"; /** * Whether the `xr-spatial-tracking` permission policy is allowed in the current document. * WebXR APIs (isSessionSupported / requestSession) require it; an embedded page/iframe must * be granted it via the iframe `allow` attribute (delegated from a parent that has it) and/or * a `Permissions-Policy` response header. Returns true when the policy API is unavailable * (can't determine → don't block, preserve prior behavior). * @link https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/iframe#allow */ function isXrSpatialTrackingAllowed(): boolean { try { const doc = globalThis.document as (Document & { permissionsPolicy?: { allowsFeature(f: string): boolean }, featurePolicy?: { allowsFeature(f: string): boolean } }) | undefined; const policy = doc?.permissionsPolicy ?? doc?.featurePolicy; if (policy && typeof policy.allowsFeature === "function") { return policy.allowsFeature("xr-spatial-tracking"); } } catch { /* ignore — fall through to default */ } return true; } /** @link https://developer.mozilla.org/en-US/docs/Web/API/XRFrame/fillPoses */ declare type FillPosesFunction = (spaces: IterableIterator, referenceSpace: XRSpace, targetArray: Float32Array) => void; declare type NeedleXRFrame = XRFrame & { fillPoses?: FillPosesFunction }; /** NeedleXRSession event argument. * Use `args.xr` to access the NeedleXRSession */ export type NeedleXREventArgs = { readonly xr: NeedleXRSession } export type SessionChangedEvt = (args: NeedleXREventArgs) => void; export type SessionRequestedEvent = (args: { readonly mode: XRSessionMode, readonly init: XRSessionInit, readonly context: Context }) => void; export type SessionRequestedEndEvent = (args: { readonly mode: XRSessionMode, readonly init: XRSessionInit, newSession: XRSession | null }) => void; /** Result of a XR hit-test * @property {XRHitTestResult} hit The original XRHitTestResult * @property {Vector3} position The hit position in world space * @property {Quaternion} quaternion The hit rotation in world space */ export type NeedleXRHitTestResult = { readonly hit: XRHitTestResult, readonly position: Vector3, readonly quaternion: Quaternion }; const debug = getParam("debugwebxr"); const debugFPS = getParam("stats"); let debugFPSFramesSinceLastUpdate = 0; // TODO: move this into the IComponent interface!? export interface INeedleXRSessionEventReceiver extends Pick { get activeAndEnabled(): boolean; supportsXR?(mode: XRSessionMode): boolean; /** Called before requesting a XR session */ onBeforeXR?(mode: XRSessionMode, args: XRSessionInit): void; onEnterXR?(args: NeedleXREventArgs): void; onUpdateXR?(args: NeedleXREventArgs): void; onLeaveXR?(args: NeedleXREventArgs): void; onXRControllerAdded?(args: NeedleXRControllerEventArgs): void; onXRControllerRemoved?(args: NeedleXRControllerEventArgs): void; } /** Contains a reference to the currently active webxr session and the controller that has changed */ export type NeedleXRControllerEventArgs = NeedleXREventArgs & { controller: NeedleXRController, change: "added" | "removed" } /** Event Arguments when a controller changed event is invoked (added or removed) * Access the controller via `args.controller`, the `args.change` property indicates if the controller was added or removed */ export type ControllerChangedEvt = (args: NeedleXRControllerEventArgs) => void; function getDOMOverlayElement(domElement: HTMLElement) { let arOverlayElement: HTMLElement | null = null; // for react cases we dont have an Engine Element const element: any = domElement; if (element.getAROverlayContainer) arOverlayElement = element.getAROverlayContainer(); else arOverlayElement = domElement; return arOverlayElement; } handleSessionGranted(); async function handleSessionGranted() { let defaultMode: XRSessionMode = "immersive-vr"; try { // If xr-spatial-tracking isn't allowed in this document (e.g. an embed whose page/iframe // wasn't granted it), XR can't run here — skip the probe so we don't log a // Permissions-Policy violation on load. if (!isXrSpatialTrackingAllowed()) return; // In app clips we default to AR if (DeviceUtilities.isNeedleAppClip()) { defaultMode = "immersive-ar"; } // Check if VR is even supported, otherwise try AR else if (!(await navigator.xr?.isSessionSupported("immersive-vr"))) { defaultMode = "immersive-ar"; } // Check if AR is supported, otherwise we can't do anything if (!(await navigator.xr?.isSessionSupported("immersive-ar")) && defaultMode === "immersive-ar") { // console.warn("[NeedleXRSession:granted] Neither VR nor AR supported, aborting session start."); // showBalloonMessage("NeidleXRSession: Neither VR nor AR supported, aborting session start."); return; } } catch (e) { console.debug("[NeedleXRSession:granted] Error while checking XR support:", e); // showBalloonWarning("NeedleXRSession: Error while checking XR support: " + (e as Error).message); return; } // TODO: asap session granted doesnt handle the pre-room yet if (getParam("debugasap")) { let asapSession = globalThis["needle:XRSession"] as XRSession | undefined | Promise; // @ts-ignore // getting TS2848 here, not sure why if (asapSession instanceof Promise) { delete globalThis["needle:XRSession"]; ContextRegistry.addContextCreatedCallback(async cb => { if (!asapSession) return; // TODO: add support to pass this to the temporary room enableSpatialConsole(true); const session = await asapSession; if (session) { const sessionInit = NeedleXRSession.getDefaultSessionInit(defaultMode); NeedleXRSession.setSession(defaultMode, session, sessionInit, cb.context); } else { console.error("[NeedleXRSession:granted] ASAP session was rejected"); } asapSession = undefined; }); return; } } // console.log("Attaching sessiongranted handler...", { haveXR: 'xr' in navigator }); // setTimeout(() => console.log("Session Granted handler attached.", { haveXR: 'xr' in navigator }), 1000); if ('xr' in navigator) { // WebXRViewer (based on Firefox) has a bug where addEventListener // throws a silent exception and aborts execution entirely. if (/WebXRViewer\//i.test(navigator.userAgent)) { console.warn('WebXRViewer does not support addEventListener'); return; } navigator.xr?.addEventListener('sessiongranted', async () => { // enableSpatialConsole(true); const lastSessionMode = sessionStorage.getItem("needle_xr_session_mode") as XRSessionMode; const lastSessionInit = sessionStorage.getItem("needle_xr_session_init") ?? null; const init = lastSessionInit ? JSON.parse(lastSessionInit) : null; let info: SessionInfo | null = null; if (contextIsLoading()) { await TemporaryXRContext.start(lastSessionMode || defaultMode, init || NeedleXRSession.getDefaultSessionInit(defaultMode)) .catch(e => console.warn("[NeedleXRSession:granted] TemporaryXRContext start failed:", e)); await waitForContextLoadingFinished(); info = await TemporaryXRContext.handoff(); } if (info) { NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current); } else if (lastSessionMode && lastSessionInit) { console.log("[NeedleXRSession:granted] Restore last session") const init = JSON.parse(lastSessionInit); NeedleXRSession.start(lastSessionMode as XRSessionMode, init).catch(e => console.warn(e)); } else { // if no session was found we start VR by default NeedleXRSession.start(defaultMode).catch(e => console.warn("[NeedleXRSession:granted] failed:", e)); } // make sure we only subscribe to the event once }, { once: true }); } else { // showBalloonWarning("NeedleXRSession: WebXR not available in this browser."); } } function saveSessionInfo(mode: XRSessionMode, init: XRSessionInit) { sessionStorage.setItem("needle_xr_session_mode", mode); sessionStorage.setItem("needle_xr_session_init", JSON.stringify(init)); } function deleteSessionInfo() { sessionStorage.removeItem("needle_xr_session_mode"); sessionStorage.removeItem("needle_xr_session_init"); } const contexts_loading: Set = new Set(); ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async cb => { contexts_loading.add(cb.context); }); ContextRegistry.registerCallback(ContextEvent.ContextCreated, async cb => { contexts_loading.delete(cb.context); const autostart = cb.context?.domElement.getAttribute("autostart") || null; handleAutoStart(autostart); }); function contextIsLoading() { return contexts_loading.size > 0; } function waitForContextLoadingFinished(): Promise { return new Promise(res => { const startTime = Date.now(); const interval = setInterval(() => { if (!contextIsLoading() || Date.now() - startTime > 60000) { clearInterval(interval); res(); } }, 100); }); } if (typeof window !== "undefined" && DeviceUtilities.isDesktop() && isDevEnvironment()) { window.addEventListener("keydown", (evt) => { if (evt.key === "x" || evt.key === "Escape") { if (NeedleXRSession.active) { NeedleXRSession.stop(); } } }); } function handleAutoStart(value: string | null) { if (!value) return; switch (value?.toLowerCase()) { case "ar": Application.registerWaitForInteraction(() => { // TODO: move WebARSessionRoot to core NeedleXRSession.start("ar"); }) break; } } // if (getParam("simulatewebxrloading")) { // ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async _cb => { // await delay(3000); // setTimeout(async () => { // const info = await TemporaryXRContext.handoff(); // if (info) NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current); // else // NeedleXRSession.start("immersive-vr") // }, 6000) // }); // let triggered = false; // window.addEventListener("click", () => { // if (triggered) return; // triggered = true; // TemporaryXRContext.start("immersive-vr", NeedleXRSession.getDefaultSessionInit("immersive-vr")); // }); // } const $initialFov = Symbol("initial-fov"); const $initialNear = Symbol("initial-near"); /** * This class manages an XRSession to provide helper methods and events. It provides easy access to the XRInputSources (controllers and hands) * - Start a XRSession with `NeedleXRSession.start(...)` * - Stop a XRSession with `NeedleXRSession.stop()` * - Access a running XRSession with `NeedleXRSession.active` * * If a XRSession is active you can use all XR-related event methods on your components to receive XR events e.g. `onEnterXR`, `onUpdateXR`, `onLeaveXR` * ```ts * export class MyComponent extends Behaviour { * // callback invoked whenever the XRSession is started or your component is added to a scene with an active XRSession * onEnterXR(args: NeedleXREventArgs) { * console.log("Entered XR"); * // access the NeedleXRSession via args.xr * } * // callback invoked whenever a controller is added (or you switch from controller to hand tracking) * onControllerAdded(args: NeedleXRControllerEventArgs) { } * } * ``` * * ### XRRig * The XRRig can be accessed via the `rig` property * Set a custom XRRig via `NeedleXRSession.addRig(...)` or `NeedleXRSession.removeRig(...)` * By default the active XRRig with the highest priority in the scene is used * * ### Screenshots in XR * Screenshots work automatically during XR sessions, including AR camera feed compositing. See {@link screenshot2} for more information. * * @category XR * @see {@link screenshot2} for taking screenshots in XR sessions */ export class NeedleXRSession implements INeedleXRSession { private static _sync: NeedleXRSync | null = null; static getXRSync(context: Context) { if (!this._sync) this._sync = new NeedleXRSync(context); return this._sync; } static get currentSessionRequest(): XRSessionMode | null { return this._currentSessionRequestMode; } private static _currentSessionRequestMode: XRSessionMode | null = null; /** Base URL for branded Needle AppClip experiences */ private static readonly appClipBaseUrl = "https://appclip.needle.tools/x/"; private static _appClipUrl: string | null = null; /** * Optionally provide a custom Needle AppClip experience to open on iOS instead of the default Needle AppClip. * This enables custom App Clip card branding (your own title and image) when launching AR on iPhone and iPad. * * You can provide either: * - the full URL, e.g. `https://appclip.needle.tools/x/your-experience` * - or just the experience id, e.g. `your-experience` (expanded to `https://appclip.needle.tools/x/your-experience`) * * **IMPORTANT**: the resulting URL must exactly match an App Clip experience URL you have registered (with custom card metadata). * For this reason the URL is opened as-is, without appending any query parameters. * * Note: custom App Clip branding requires an active PRO subscription. See the iOS WebXR App Clip docs (Custom Branding for iOS AR). * @link https://engine.needle.tools/docs/how-to-guides/xr/ios-webxr-app-clip#custom-branding-for-ios-ar */ static get appClipUrl(): string | null { return this._appClipUrl; } static set appClipUrl(value: string | null) { if (!value) { this._appClipUrl = null; return; } const trimmed = value.trim(); // accept a full URL as-is, otherwise treat the value as an experience id and expand it using the base url this._appClipUrl = /^https?:\/\//i.test(trimmed) ? trimmed : this.appClipBaseUrl + trimmed.replace(/^\/+/, ""); } /** * @internal * Returns the AppClip launch URL to open (or prefetch) on iOS for the current page. * When a custom {@link appClipUrl} is set this is the branded experience URL (opened as-is so it matches the registered URL); * otherwise it is the default Needle AppClip launch URL with the current page forwarded via the `url` parameter. */ static getAppClipLaunchUrl(): URL { if (this._appClipUrl) return new URL(this._appClipUrl); // Forward to the default Needle AppClip experience (using the apple.com url the appclip overlay shows immediately) // alternative: `https://appclip.needle.tools/ar?url=${location.href}` const url = new URL("https://appclip.apple.com/id?p=tools.needle.launch-app.Clip"); url.searchParams.set("url", location.href); return url; } /** * @returns the active @type {NeedleXRSession} (if any active) or null */ static get active(): NeedleXRSession | null { return this._activeSession; } /** The active xr session mode (if any xr session is active) * @link https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode */ static get activeMode() { return this._activeSession?.mode ?? null; } /** XRSystem via navigator.xr access * @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem */ static get xrSystem(): XRSystem | undefined { return ('xr' in navigator) ? navigator.xr : undefined; } /** * @returns true if the browser supports WebXR (`immersive-vr` or `immersive-ar`) * @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported */ static isXRSupported() { return Promise.all([this.isVRSupported(), this.isARSupported()]).then(res => res.some(e => e)).catch(() => false); } /** * @returns true if the browser supports immersive-vr (WebXR) * @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported */ static isVRSupported() { return this.isSessionSupported("immersive-vr"); } /** * @returns true if the browser supports immersive-ar (WebXR) * @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported */ static isARSupported() { return this.isSessionSupported("immersive-ar"); } /** * @param mode The XRSessionMode to check if it is supported * @returns true if the browser supports the given XRSessionMode */ static isSessionSupported(mode: XRSessionMode): Promise { // Re-probe when XR devices change (e.g. a headset connects). this.ensureDeviceChangeInvalidation(); // Return the RESOLVED boolean once we have it. This cache stores booleans (not promises) // on purpose: some call sites read it synchronously — e.g. the iOS AppClip path in start() // must decide within the user gesture, so it can't await here. const resolved = this._sessionSupportedCache[mode]; if (resolved !== undefined) return Promise.resolve(resolved); // De-duplicate concurrent probes: a burst of callers in the same tick (XR buttons, // offer-session, support checks) shares ONE underlying navigator.xr.isSessionSupported() // call per mode instead of each re-running the `xr-spatial-tracking` Permissions-Policy check. const inflight = this._sessionSupportedInFlight[mode]; if (inflight !== undefined) return inflight; // No WebXR API available right now: report unsupported but do NOT cache it. A WebXR // polyfill or emulator (iwer, the WebXR browser extension) may install navigator.xr // AFTER engine init, and until it exists there is no devicechange event that could // invalidate a cached false. const xrSystem = this.xrSystem; if (!xrSystem) return Promise.resolve(false); let result: Promise; // isSessionSupported() requires the `xr-spatial-tracking` permission policy to be // allowed in this document. In an embed whose page/iframe wasn't granted it, calling // the API logs a Permissions-Policy violation and can't succeed — so skip the call // and report "unsupported" instead of spamming the console. // See https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/iframe#allow if (!isXrSpatialTrackingAllowed()) { if (debug) console.warn("[WebXR] xr-spatial-tracking is not allowed in this document — reporting XR as unsupported. The embedding page/iframe must grant the policy."); result = Promise.resolve(false); } else { result = xrSystem.isSessionSupported(mode) .catch(err => { if (debug) console.error(err); return false; }); } this._sessionSupportedInFlight[mode] = result; result.then(v => { this._sessionSupportedCache[mode] = v; delete this._sessionSupportedInFlight[mode]; }); return result; } /** Resolved support results, kept as booleans so call sites can read them synchronously (e.g. the iOS AppClip path). */ private static _sessionSupportedCache: { [mode in XRSessionMode]?: boolean } = {}; /** In-flight probes, to de-duplicate concurrent isSessionSupported() calls per mode. */ private static _sessionSupportedInFlight: { [mode in XRSessionMode]?: Promise } = {}; private static _deviceChangeHooked = false; /** Invalidate the support cache when the set of XR devices changes, so a newly * connected headset is picked up by the next isSessionSupported() call. */ private static ensureDeviceChangeInvalidation() { if (this._deviceChangeHooked) return; const xr = this.xrSystem; // navigator.xr may not exist yet (polyfill/emulator installed later) — retry on the next call. if (!xr) return; this._deviceChangeHooked = true; try { xr.addEventListener?.("devicechange", () => { this._sessionSupportedCache = {}; this._sessionSupportedInFlight = {}; }); } catch { /* xrSystem may not support events; cache simply won't be invalidated */ } } private static _currentSessionRequest?: Promise; private static _activeSession: NeedleXRSession | null; /** Register to listen to XRSession start events. Unsubscribe with `offXRSessionStart` */ static onSessionRequestStart(evt: SessionRequestedEvent) { this._sessionRequestStartListeners.push(evt); } /** Unsubscribe from request start evt. Register with `onSessionRequestStart` */ static offSessionRequestStart(evt: SessionRequestedEvent) { const index = this._sessionRequestStartListeners.indexOf(evt); if (index >= 0) this._sessionRequestStartListeners.splice(index, 1); } private static readonly _sessionRequestStartListeners: SessionRequestedEvent[] = []; /** Called after the session request has finished */ static onSessionRequestEnd(evt: SessionRequestedEndEvent) { this._sessionRequestEndListeners.push(evt); } /** Unsubscribe from request end evt */ static offSessionRequestEnd(evt: SessionRequestedEndEvent) { const index = this._sessionRequestEndListeners.indexOf(evt); if (index >= 0) this._sessionRequestEndListeners.splice(index, 1); } private static readonly _sessionRequestEndListeners: SessionRequestedEndEvent[] = []; /** Listen to XR session started. Unsubscribe with `offXRSessionStart` */ static onXRSessionStart(evt: SessionChangedEvt) { this._xrStartListeners.push(evt); }; /** Unsubscribe from XRSession started events */ static offXRSessionStart(evt: SessionChangedEvt) { const index = this._xrStartListeners.indexOf(evt); if (index >= 0) this._xrStartListeners.splice(index, 1); } private static readonly _xrStartListeners: SessionChangedEvt[] = []; /** Listen to XR session ended. Unsubscribe with `offXRSessionEnd` */ static onXRSessionEnd(evt: SessionChangedEvt) { this._xrEndListeners.push(evt); }; /** Unsubscribe from XRSession started events */ static offXRSessionEnd(evt: SessionChangedEvt) { const index = this._xrEndListeners.indexOf(evt); if (index >= 0) this._xrEndListeners.splice(index, 1); } private static readonly _xrEndListeners: SessionChangedEvt[] = []; /** Listen to controller added events. * Events are cleared when starting a new session **/ static onControllerAdded(evt: ControllerChangedEvt) { this._controllerAddedListeners.push(evt); } /** Unsubscribe from controller added evts */ static offControllerAdded(evt: ControllerChangedEvt) { const index = this._controllerAddedListeners.indexOf(evt); if (index >= 0) this._controllerAddedListeners.splice(index, 1); } private static readonly _controllerAddedListeners: ControllerChangedEvt[] = []; /** Listen to controller removed events * Events are cleared when starting a new session **/ static onControllerRemoved(evt: ControllerChangedEvt) { this._controllerRemovedListeners.push(evt); } /** Unsubscribe from controller removed events */ static offControllerRemoved(evt: ControllerChangedEvt) { const index = this._controllerRemovedListeners.indexOf(evt); if (index >= 0) this._controllerRemovedListeners.splice(index, 1); } private static readonly _controllerRemovedListeners: ControllerChangedEvt[] = []; /** If the browser supports offerSession - creating a VR or AR button in the browser navigation bar */ static offerSession(mode: XRSessionMode, init: XRSessionInit | "default", context: Context): boolean { if ('xr' in navigator && navigator.xr && 'offerSession' in navigator.xr) { if (typeof navigator.xr.offerSession === "function") { console.log("WebXR offerSession is available - requesting mode: " + mode); if (init == "default") { init = this.getDefaultSessionInit(mode); } navigator.xr.offerSession(mode, { ...init }).then((session) => { return NeedleXRSession.setSession(mode, session, init as XRSessionInit, context); }).catch(_ => { console.log("XRSession offer rejected (perhaps because another call to offerSession was made or a call to requestSession was made)") }); } return true; } return false; } /** @returns a new XRSession init object with defaults */ static getDefaultSessionInit(mode: Omit): XRSessionInit { switch (mode) { case "immersive-ar": const arFeatures = ['anchors', 'local-floor', 'layers', 'dom-overlay', 'hit-test', 'unbounded']; // Don't request handtracking by default on VisionOS if (!DeviceUtilities.isVisionOS()) arFeatures.push('hand-tracking'); return { optionalFeatures: arFeatures, } case "immersive-vr": const vrFeatures = ['local-floor', 'bounded-floor', 'high-fixed-foveation-level', 'layers']; // Don't request handtracking by default on VisionOS if (!DeviceUtilities.isVisionOS()) vrFeatures.push('hand-tracking'); return { optionalFeatures: vrFeatures, } default: console.warn("No default session init for mode", mode); return {}; } } /** start a new webXR session (make sure to stop already running sessions before calling this method) * @param mode The XRSessionMode to start (e.g. `immersive-vr` or `immersive-ar`) or `ar` to start `immersive-ar` on supported devices OR on iOS devices it will export an interactive USDZ and open in Quicklook. * Get more information about WebXR modes: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode * @param init The XRSessionInit to use (optional), docs: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionInit * @param context The Needle Engine context to use */ static async start(mode: XRSessionMode | "ar" | "quicklook", init?: XRSessionInit, context?: Context): Promise { // setup session init args, make sure we have default values if (!init) init = {}; // handle iOS platform where "immersive-ar" is special: // - we either launch QuickLook // - or forward to the Needle App Clip experience for WebXR AR if (DeviceUtilities.isiOS()) { // IMPORTANT: on iOS we should have prefetched immersive-ar support already on app start (this is done in xrInit()) - if not then clicking the Enter AR button for the very first time does not immediately open the AppClip overlay. // Without an async await here the overlay *does* however open even when clicking for the very first time. (To test this you need to clear the local experience cache on device) const arSupported = this._sessionSupportedCache["immersive-ar"] ?? await this.isARSupported(); // On VisionOS, we use QuickLook for AR experiences; no AppClip support for now. if (DeviceUtilities.isVisionOS() && !arSupported && (mode === "ar" || mode === "immersive-ar")) { mode = "quicklook"; } if (mode === "quicklook") { Telemetry.sendEvent(Context.Current, "xr", { action: "quicklook_export", source: "NeedleXRSession.start", }); InternalUSDZRegistry.exportAndOpen(); return null; } if (!arSupported && (mode === "immersive-ar" || mode === "ar")) { this.invokeSessionRequestStart("immersive-ar", init, context ?? Context.Current); // if we are in an iframe, we need to navigate the top window const topWindow = window.top || window; // Forward to the AppClip experience. By default this is the Needle AppClip; a custom branded // experience can be set via NeedleXRSession.appClipUrl (which is then opened as-is so it matches the registered URL). const url = NeedleXRSession.getAppClipLaunchUrl(); const urlStr = url.toString(); Telemetry.sendEvent(Context.Current, "xr", { action: "app_clip_launch", source: "NeedleXRSession.start", url: urlStr, }); try { console.debug("iOS device detected - opening Needle App Clip for AR experience", { mode, init, url }); // Navigate to the App Clip invocation URL synchronously, INSIDE the tap's user // activation. iOS only presents the App Clip card for a navigation that happens // within the user gesture — the previous prewarm sequence (navigate to a // `?prewarm=1` variant, await 300ms, navigate again) pushed the real invocation // out of the gesture, which is one reason the card regularly needed a second tap. // The invocation itself makes iOS fetch the experience metadata, so on a cold // metadata cache the card may still need a second tap (the button shows // "Tap again" for that case), but a warm cache now presents on the first tap. // eslint-disable-next-line xss/no-location-href-assign topWindow.location.href = urlStr; } catch (e) { console.warn(`Error navigating to AppClip ${urlStr}\n`, e); // if top window navigation fails and we are in an iframe, we try to navigate the top window directly const weAreInIframe = window !== window.top; if (weAreInIframe) { // we can try to open a new tab as a fallback window.open(urlStr, "_blank"); } // eslint-disable-next-line xss/no-location-href-assign else window.location.href = urlStr; } setTimeout(() => { this.invokeSessionRequestEnd("immersive-ar", init || {}, null); }, 3000); return null; } } if (mode === "quicklook") { console.error("QuickLook mode is only supported on iOS devices."); return null; } // Since we now know we are not on iOS, ar mode becomes "immersive-ar" if (mode == "ar") { mode = "immersive-ar"; } if (isDevEnvironment() && getParam("debugxrpreroom")) { console.warn("Debug: Starting temporary XR session"); await TemporaryXRContext.start(mode, init || NeedleXRSession.getDefaultSessionInit(mode)); return null; } if (this._currentSessionRequest) { console.warn("A XRSession is already being requested"); if (debug || isDevEnvironment()) showBalloonWarning("A XRSession is already being requested"); return this._currentSessionRequest.then(() => this._activeSession!); } if (this._activeSession) { console.error("A XRSession is already running"); return this._activeSession; } // Make sure we have a context if (!context) context = Context.Current; if (!context) context = ContextRegistry.All[0] as Context; if (!context) throw new Error("No Needle Engine Context found"); //performance.mark('NeedleXRSession start'); switch (mode) { // Setup VR initialization parameters case "immersive-ar": { const supported = await NeedleXRSession.isSessionSupported('immersive-ar') if (supported !== true) { console.error(mode + ' is not supported by this browser.'); return null; } const defaultInit: XRSessionInit = this.getDefaultSessionInit(mode); const domOverlayElement = getDOMOverlayElement(context.domElement); if (domOverlayElement && !DeviceUtilities.isQuest()) { // excluding quest since dom-overlay breaks sessiongranted // 'dom-overlay' is already part of the default optional features — only the root needs to be assigned defaultInit.domOverlay = { root: domOverlayElement }; } init = { ...defaultInit, ...init, } } break; // Setup AR initialization parameters case "immersive-vr": { const supported = await NeedleXRSession.isSessionSupported('immersive-vr') if (supported !== true) { console.error(mode + ' is not supported by this browser.'); return null; } const defaultInit: XRSessionInit = this.getDefaultSessionInit(mode); init = { ...defaultInit, ...init, } } break; default: console.warn("No default session init for mode", mode); break; } // Fix: define these here because VariantLauncher crashes otherwise. Spec: https://immersive-web.github.io/webxr/#feature-dependencies // Issue: https://linear.app/needle/issue/NE-5136 init.optionalFeatures ??= []; init.requiredFeatures ??= []; // we stop a temporary session here (if any runs) await TemporaryXRContext.stop(); const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr if (debug) console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;", init, scripts); else console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;"); for (const script of scripts) { if (script.onBeforeXR && script.activeAndEnabled && !script.destroyed) { script.onBeforeXR(mode, init); } } this.invokeSessionRequestStart(mode, init, context ?? Context.Current); // Features are collected from multiple places (default init, user init, component onBeforeXR // callbacks, session request hooks) — make sure the final request contains no duplicates and // no optional feature that is already required const requiredFeatures = Array.from(new Set(init.requiredFeatures)); const optionalFeatures = Array.from(new Set(init.optionalFeatures)).filter(f => !requiredFeatures.includes(f)); init.requiredFeatures = requiredFeatures; init.optionalFeatures = optionalFeatures; if (debug) showBalloonMessage("Requesting " + mode + " session (" + Date.now() + ")"); Telemetry.sendEvent(Context.Current, "xr", { action: "session_request", mode: mode, features: ((init.requiredFeatures ?? []).concat(init.optionalFeatures ?? [])).join(","), source: "NeedleXRSession.start", }); this._currentSessionRequest = navigator?.xr?.requestSession(mode, init); this._currentSessionRequestMode = mode; /**@type {XRSystem} */ const newSession = await (this._currentSessionRequest)?.catch(e => { console.error(e, "Code: " + e?.code, "requiredFeatures:", init.requiredFeatures, "optionalFeatures:", init.optionalFeatures); if (e?.code === 9) { // NotSupportedError: surface the platform's reason if it provides one // (e.g. which required feature is unsupported) instead of only a generic hint let reason = typeof e?.message === "string" && e.message.length > 0 ? e.message : "Make sure you allow the required permissions."; if (!init.requiredFeatures?.length) { // All requested features are optional, so the session *configuration* can not be // the reason for the rejection (the platform message claiming so is misleading): // the browser or device is blocking the session mode itself reason = mode === "immersive-ar" ? "AR seems to be blocked or unavailable on this device. Check the browser's site settings (Augmented Reality and Camera permissions) and that 'Google Play Services for AR' is installed and up to date." : "VR seems to be blocked or unavailable on this device. Check the browser's site settings (Virtual Reality permission)."; if (window.self !== window.top) reason += " Embedded pages need allow=\"xr-spatial-tracking; camera\" on the iframe."; console.warn(`[Needle] The ${mode} session request was rejected although all requested features are optional — the device or browser is blocking the session mode itself, not the session configuration.`); } showBalloonWarning(`Couldn't start XR session: ${reason}`, { duration: 30_000 }); } console.log("If the specified XR configuration is not supported (e.g. entering AR doesnt work) - make sure you access the website on a secure connection (HTTPS) and your device has the required permissions (e.g. camera access)"); const notSecure = location.protocol === 'http:'; if (notSecure) showBalloonWarning("XR requires a secure connection (HTTPS)", { duration: 30_000 }); }); this._currentSessionRequest = undefined; this._currentSessionRequestMode = null; this.invokeSessionRequestEnd(mode, init, newSession); if (!newSession) { console.warn("XR Session request was rejected"); return null; } const session = this.setSession(mode, newSession, init, context); //performance.mark('NeedleXRSession end'); //performance.measure('NeedleXRSession Startup', 'NeedleXRSession start', 'NeedleXRSession end'); return session; } private static invokeSessionRequestStart(mode: XRSessionMode, init: XRSessionInit, context: Context) { for (const listener of this._sessionRequestStartListeners) { listener({ mode, init, context }); } } private static invokeSessionRequestEnd(mode: XRSessionMode, init: XRSessionInit, session: XRSession | null | undefined | void) { for (const listener of this._sessionRequestEndListeners) { listener({ mode, init, newSession: session || null }); } } static setSession(mode: XRSessionMode, session: XRSession, init: XRSessionInit, context: Context) { if (this._activeSession) { console.error("A XRSession is already running"); return this._activeSession; } const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr; this._activeSession = new NeedleXRSession(mode, session, context, { scripts: scripts, controller_added: this._controllerAddedListeners, controller_removed: this._controllerRemovedListeners, init: init }); session.addEventListener("end", this.onEnd); if (debug) console.log("%c" + `Started ${mode} session`, "font-weight:bold;", scripts); else console.log("%c" + `Started ${mode} session`, "font-weight:bold;"); return this._activeSession; } private static $_stop_request = Symbol(); /** stops the active XR session */ static stop(): void { const session = this._activeSession; if (session) { if (session[this.$_stop_request] === undefined) { if (debug) console.log("[NeedleXRSession] Stopping XR Session... (new)"); // We can not call session.end() immediately because we might be within a frame callback // For example if a use calls `NeedleXRSession.stop()` within a `update()` callback session[this.$_stop_request] = setTimeout(() => { session.end(); }); } else if (debug) { console.warn("[NeedleXRSession] XR Session stop already requested"); } } } private static onEnd = () => { if (debug) console.log("XR Session ended"); this._activeSession = null; } /** The needle engine context this session was started from */ readonly context: Context; get sync(): NeedleXRSync | null { return NeedleXRSession._sync; } /** Returns true if the xr session is still active */ get running(): boolean { return !this._ended && this.session != null; } /** * @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession */ readonly session: XRSession; /** XR Session Mode: AR or VR */ readonly mode: XRSessionMode; /** * The XRSession interface's read-only interactionMode property describes the best space (according to the user agent) for the application to draw an interactive UI for the current session. * @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/interactionMode */ get interactionMode(): "screen-space" | "world-space" { return this.session["interactionMode"]!; } /** * @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/visibilityState * @returns {XRVisibilityState} The visibility state of the XRSession */ get visibilityState(): XRVisibilityState { return this.session.visibilityState; } /** * Check if the session is `visible-blurred` - this means e.g. the keyboard is shown */ get isVisibleBlurred(): boolean { return this.session.visibilityState === 'visible-blurred' } /** * Check if the session has system keyboard support */ get isSystemKeyboardSupported(): boolean { return this.session.isSystemKeyboardSupported; } /** * @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/environmentBlendMode */ get environmentBlendMode(): XREnvironmentBlendMode { return this.session.environmentBlendMode; } /** * The current XR frame * @link https://developer.mozilla.org/en-US/docs/Web/API/XRFrame */ get frame(): NeedleXRFrame { return this.context.xrFrame! as NeedleXRFrame; } /** The currently active/connected controllers */ readonly controllers: NeedleXRController[] = []; /** shorthand to query the left controller. Use `controllers` to get access to all connected controllers */ get leftController() { return this.controllers.find(c => c.side === "left"); } /** shorthand to query the right controller. Use `controllers` to get access to all connected controllers */ get rightController() { return this.controllers.find(c => c.side === "right"); } /** @returns the given controller if it is connected */ getController(side: XRHandedness | number) { if (typeof side === "number") return this.controllers[side] || null; return this.controllers.find(c => c.side === side) || null; } /** Returns true if running in pass through mode in immersive AR (e.g. user is wearing a headset while in AR) */ get isPassThrough() { if (this.environmentBlendMode !== 'opaque' && this.interactionMode === "world-space") return true; // since we can not rely on interactionMode check we check the controllers too // https://linear.app/needle/issue/NE-4057 // the following is a workaround for the issue above if (this.mode === "immersive-ar" && this.environmentBlendMode !== 'opaque') { // if we have any tracked pointer controllers we're also in passthrough if (this.controllers.some(c => c.inputSource.targetRayMode === "tracked-pointer")) return true; } if (isDevEnvironment() && DeviceUtilities.isDesktop() && this.mode === "immersive-ar") { return true; } return false; } get isAR() { return this.mode === 'immersive-ar'; } get isVR() { return this.mode === 'immersive-vr'; } /** If the AR mode is not immersive (meaning the user is e.g. holding a phone instead of wearing a AR passthrough headset) */ get isScreenBasedAR() { return this.isAR && !this.isPassThrough; } get posePosition() { return this._transformPosition; } get poseOrientation() { return this._transformOrientation; } /** @returns the context.renderer.xr.getReferenceSpace() result */ get referenceSpace(): XRSpace | null { return this.context.renderer.xr.getReferenceSpace(); } /** @returns the XRFrame `viewerpose` using the xr `referenceSpace` */ get viewerPose(): XRViewerPose | undefined { return this._viewerPose; } /** @returns `true` if any image is currently being tracked or emulated */ get isTrackingImages() { if (this.frame && "getImageTrackingResults" in this.frame && typeof this.frame.getImageTrackingResults === "function") { try { const trackingResult = this.frame.getImageTrackingResults(); for (const result of trackingResult) { const state = result.trackingState; if (state === "tracked" || state === "emulated") { return true; } } } catch { // Looks like we get a NotSupportedException on Android since the method is known // but the feature is not supported by the session // TODO Can we check here if we even requested the image-tracking feature instead of catching? return false; } } return false; } /** The currently active XR rig */ get rig(): IXRRig | null { const rig = this._rigs[0] ?? null; if (rig?.gameObject && isDestroyed(rig.gameObject) || rig?.isActive === false) { this.updateActiveXRRig(); return this._rigs[0] ?? null; } return rig; } private _rigScale: number = 1; private _lastRigScaleUpdate: number = -1; /** Get the XR Rig worldscale. * * **For AR** * If you want to modify the scale in AR at runtime get the WebARSessionRoot component via `findObjectOfType(WebARSessionRoot)` and then set the `arScale` value. * @returns the scale of the XR rig * */ get rigScale(): number { // const ar_scaleValue = this.context.domElement.getAttribute("ar-scale"); // if (ar_scaleValue) { // const scale = parseFloat(ar_scaleValue); // if (!isNaN(scale)) return scale; // } if (!this._rigs[0]) return 1; if (this._lastRigScaleUpdate !== this.context.time.frame) { this._lastRigScaleUpdate = this.context.time.frame; this._rigScale = this._rigs[0].gameObject.worldScale.x; } return this._rigScale; } /** add a rig to the available XR rigs - if it's priority is higher than the currently active rig it will be enabled */ addRig(rig: IXRRig) { const i = this._rigs.indexOf(rig); if (i >= 0) return; if (rig.priority === undefined) rig.priority = 0; this._rigs.push(rig); this.updateActiveXRRig(); } /** Remove a rig from the available XR Rigs */ removeRig(rig: IXRRig) { const i = this._rigs.indexOf(rig); if (i === -1) return; this._rigs.splice(i, 1); this.updateActiveXRRig(); } /** Sets a XRRig to be active which will parent the camera to this rig */ setRigActive(rig: IXRRig) { const i = this._rigs.indexOf(rig); const currentlyActive = this._rigs[0]; this._rigs.splice(i, 1); this._rigs.unshift(rig); // if there's another rig currently active we need to make sure we have at least the same priority rig.priority = currentlyActive?.priority ?? 0; this.updateActiveXRRig(); } /** * @returns the user position in the rig space */ getUserOffsetInRig() { const positionInRig = this.context.mainCamera?.position; if (!positionInRig || !this.rig) return getTempVector(0, 0, 0); const vec = getTempVector(positionInRig); vec.x *= -1; vec.z *= -1; vec.applyQuaternion(getTempQuaternion(this.rig.gameObject.quaternion)); return vec; } private updateActiveXRRig() { const previouslyActiveRig = this._rigs[0] ?? null; // ensure that the default rig is in the scene if (this._defaultRig.gameObject.parent !== this.context.scene) this.context.scene.add(this._defaultRig.gameObject); // ensure the fallback rig is always active!!! this._defaultRig.gameObject.visible = true; // ensure that the default rig is in the list of available rigs if (!this._rigs.includes(this._defaultRig)) this._rigs.push(this._defaultRig); // find the rig with the highest priority and make sure it's at the beginning of the array let highestPriorityRig: IXRRig = this._rigs[0]; if (highestPriorityRig && highestPriorityRig.priority === undefined) highestPriorityRig.priority = 0; for (let i = 1; i < this._rigs.length; i++) { const rig = this._rigs[i]; if (!rig.isActive) continue; if (isDestroyed(rig.gameObject)) { this._rigs.splice(i, 1); i--; continue; } if (!highestPriorityRig || highestPriorityRig.isActive === false || (rig.priority !== undefined && rig.priority > highestPriorityRig.priority!)) { highestPriorityRig = rig; } } // make sure the highest priority rig is at the beginning if it isnt already if (previouslyActiveRig !== highestPriorityRig) { const index = this._rigs.indexOf(highestPriorityRig); if (index >= 0) this._rigs.splice(index, 1); this._rigs.unshift(highestPriorityRig); } if (debug) { if (previouslyActiveRig === highestPriorityRig) console.log("Updated Active XR Rig:", highestPriorityRig, "prev:", previouslyActiveRig); else console.log("Updated Active XRRig:", highestPriorityRig, " (the same as before)"); } } private _rigs: IXRRig[] = []; private _viewerHitTestSource: XRHitTestSource | null = null; /** Returns a XR hit test result (if hit-testing is available) in rig space * @param source If provided, the hit test will be performed for the given controller */ getHitTest(source?: NeedleXRController): NeedleXRHitTestResult | null { if (source) { return this.getControllerHitTest(source); } if (!this._viewerHitTestSource) return null; const hitTestSource = this._viewerHitTestSource; const hitTestResults = this.frame.getHitTestResults(hitTestSource); if (hitTestResults.length > 0) { const hit = hitTestResults[0]; return this.convertHitTestResult(hit); } return null; } private getControllerHitTest(controller: NeedleXRController): NeedleXRHitTestResult | null { const hitTestSource = controller.getHitTestSource(); if (!hitTestSource) return null; const res = this.frame.getHitTestResultsForTransientInput(hitTestSource); for (const result of res) { if (result.inputSource === controller.inputSource) { for (const hit of result.results) { return this.convertHitTestResult(hit); } } } return null; } private convertHitTestResult(result: XRHitTestResult): NeedleXRHitTestResult | null { const referenceSpace = this.context.renderer.xr.getReferenceSpace(); const pose = referenceSpace && result.getPose(referenceSpace); if (pose) { const pos = getTempVector(pose.transform.position); const rot = getTempQuaternion(pose.transform.orientation); const camera = this.context.mainCamera; if (camera?.parent !== this._cameraRenderParent) { pos.applyMatrix4(flipForwardMatrix); } if (camera?.parent) { pos.applyMatrix4(camera.parent.matrixWorld); rot.multiply(flipForwardQuaternion); // apply parent quaternion (if parent is moved/rotated) const parentRotation = getWorldQuaternion(camera.parent); // ensure that "up" (y+) is pointing away from the wall parentRotation.premultiply(flipForwardQuaternion); rot.premultiply(parentRotation); } return { hit: result, position: pos, quaternion: rot }; } return null; } /** convert a XRRigidTransform from XR session space to threejs / Needle Engine XR space */ convertSpace(transform: XRRigidTransform): { position: Vector3, quaternion: Quaternion } { const pos = getTempVector(transform.position); pos.applyMatrix4(flipForwardMatrix); const rot = getTempQuaternion(transform.orientation); rot.premultiply(flipForwardQuaternion); return { position: pos, quaternion: rot }; } /** this is the implictly created XR rig */ private readonly _defaultRig: IXRRig; /** all scripts that receive some sort of XR update event */ private readonly _xr_scripts: INeedleXRSessionEventReceiver[]; /** scripts that have onUpdateXR event methods */ private readonly _xr_update_scripts: INeedleXRSessionEventReceiver[] = []; /** scripts that are in the scene but inactive (e.g. disabled parent gameObject) */ private readonly _inactive_scripts: INeedleXRSessionEventReceiver[] = []; /** tracks scripts that have received onEnterXR — prevents spurious onLeaveXR calls */ private readonly _scripts_in_xr = new Set(); private readonly _controllerAdded: ControllerChangedEvt[]; private readonly _controllerRemoved: ControllerChangedEvt[]; private readonly _originalCameraWorldPosition?: Vector3 | null; private readonly _originalCameraWorldRotation?: Quaternion | null; private readonly _originalCameraWorldScale?: Vector3 | null; private readonly _originalCameraParent?: Object3D | null; /** we store the main camera reference here each frame to make sure we have a rendering camera * this e.g. the case when the XR rig with the camera gets disabled (and thus this.context.mainCamera is unassigned) */ private _mainCamera: ICamera | null = null; private constructor(mode: XRSessionMode, session: XRSession, context: Context, extra: { scripts: INeedleXRSessionEventReceiver[], controller_added: ControllerChangedEvt[], controller_removed: ControllerChangedEvt[], /** the initialization arguments */ init: XRSessionInit, }) { //performance.mark(measure_SessionStartedMarker); saveSessionInfo(mode, extra.init); this.session = session; this.mode = mode; this.context = context; if (debug || getParam("console")) enableSpatialConsole(true); this._xr_scripts = [...extra.scripts]; this._xr_update_scripts = this._xr_scripts.filter(e => typeof e.onUpdateXR === "function"); this._controllerAdded = extra.controller_added; this._controllerRemoved = extra.controller_removed; registerFrameEventCallback(this.onBefore, FrameEvent.LateUpdate); this.context.pre_render_callbacks.push(this.onBeforeRender); this.context.post_render_callbacks.push(this.onAfterRender); if (extra.init.optionalFeatures?.includes("hit-test") || extra.init.requiredFeatures?.includes("hit-test")) { session.requestReferenceSpace('viewer').then((referenceSpace) => { return session.requestHitTestSource?.call(session, { space: referenceSpace })?.then((source) => { return this._viewerHitTestSource = source; }).catch(e => console.error(e)); }).catch(e => console.error(e)); } if (this.context.mainCamera) { this._originalCameraWorldPosition = getWorldPosition(this.context.mainCamera, new Vector3()); this._originalCameraWorldRotation = getWorldQuaternion(this.context.mainCamera, new Quaternion()); this._originalCameraWorldScale = getWorldScale(this.context.mainCamera, new Vector3()); this._originalCameraParent = this.context.mainCamera.parent; if (this.context.mainCamera instanceof PerspectiveCamera) { this.context.mainCamera[$initialFov] = this.context.mainCamera.fov; // if (this.mode === "immersive-ar" && this.context.mainCamera.near > .1) { // console.log("[WebXR] Setting near plane to 0.1 for better AR experience (was " + this.context.mainCamera.near + "). The initial near plane will be restored when the session ends."); // this.context.mainCamera.near = 0.1; // this.context.mainCamera[$initialNear] = this.context.mainCamera.near; // } } } this._defaultRig = new ImplictXRRig(); this.context.scene.add(this._defaultRig.gameObject); this.addRig(this._defaultRig); // register already connected input sources // this is for when the session is already running (via a temporary xr session) // and the controllers are already connected for (let i = 0; i < session.inputSources.length; i++) { const inputSource = session.inputSources[i]; if (!inputSource.handedness) { console.warn("Input source in xr session has no handedness - ignoring", i); continue; } this.onInputSourceAdded(inputSource); } // handle controller and input source changes changes this.session.addEventListener('end', this.onEnd); // handle input sources change this.session.addEventListener("inputsourceschange", /* @ts-ignore (ignore CI XRInputSourceChangeEvent mismatch) */ (evt: XRInputSourcesChangeEvent) => { // handle removed controllers for (const removedInputSource of evt.removed) { this.disconnectInputSource(removedInputSource); } for (const newInputSource of evt.added) { this.onInputSourceAdded(newInputSource); } }); // Unfortunately the code below doesnt work: the session never receives any input sources sometimes // https://developer.mozilla.org/en-US/docs/Web/API/XRSession/visibilitychange_event // this.session.addEventListener("visibilitychange", (evt: XRSessionEvent) => { // // sometimes when entering an XR session the controllers are not added/not in the list and we don't receive an event // // this is a workaround trying to add controllers when the scene visibility changes to "visible" // // e.g. due to a user opening and closing the menu // if (this.controllers.length === 0 && evt.session.visibilityState === "visible") { // for (const controller of evt.session.inputSources) { // this.onInputSourceAdded(controller); // } // } // }) // we set the session on the webxr manager at the end because we want to receive inputsource events first // e.g. in case there's a bug in the threejs codebase this.context.xr = this; // Since three r185 the WebXRManager uses the XRProjectionLayer path whenever the (native) // XRWebGLBinding exposes `createProjectionLayer`, constructing `new XRWebGLBinding(session, gl)`. // The native constructor requires a NATIVE XRSession and rejects a polyfilled one with a // TypeError ("parameter 1 is not of type 'XRSession'"). WebXR emulators / polyfills (Meta // immersive-web-emulator, webxr-polyfill, ...) hand three exactly such a non-native session. // We cannot detect this up front (a polyfill replaces the global XRSession, so the polyfilled // session *is* an instanceof the polyfill's XRSession), so instead we react to the actual error: // real devices succeed on the first try and never touch the code below. Only when setSession // throws that specific TypeError do we retry once with the layers path disabled, letting three // fall back to XRWebGLLayer which the emulator/polyfill supports. // See https://github.com/meta-quest/immersive-web-emulator/issues/80 const trySetSession = () => this.context.renderer.xr.setSession(this.session); trySetSession().then(this.onRendererSessionSet) .catch(err => { const glBindingProto = (typeof XRWebGLBinding !== "undefined" ? XRWebGLBinding.prototype : null) as unknown as (Record | null); if (err instanceof TypeError && glBindingProto && "createProjectionLayer" in glBindingProto) { // Emulator/polyfill: disable the projection-layer path and retry once. console.warn("renderer.xr.setSession failed constructing XRWebGLBinding (non-native XRSession from a WebXR emulator/polyfill) - retrying with the XRProjectionLayer path disabled. See https://github.com/meta-quest/immersive-web-emulator/issues/80"); const savedCreateProjectionLayer = glBindingProto.createProjectionLayer; delete glBindingProto.createProjectionLayer; void trySetSession().then(this.onRendererSessionSet) .catch(retryErr => console.error("renderer.xr.setSession retry failed:", retryErr)) .finally(() => { glBindingProto.createProjectionLayer = savedCreateProjectionLayer; }); return; } console.error("renderer.xr.setSession failed:", err); return; }); // disable three.js renderer controller autoUpdate (added in ac67b31e3548386f8a93e23a4176554c92bbd0d9) if ("controllerAutoUpdate" in this.context.renderer.xr) { console.debug("Disabling three.js controllerAutoUpdate"); this.context.renderer.xr.controllerAutoUpdate = false; } else if (debug) { console.warn("controllerAutoUpdate is not available in three.js - cannot disable it"); } // In the appclip we want to make sure the canvas resolution is correctly applied with the DPR // this change has been added to Needle Go AppClip in daa9c186e49fe9d6f14d200e307eb0664e35ad72 // But for an immediate fix for Needle Engine projects we also add it here. // We can most likely remove the folllowing block in march 2026 if (DeviceUtilities.isNeedleAppClip()) { window.requestAnimationFrame(() => { const canvas = this.context.renderer.domElement; const dpr = window.devicePixelRatio || 1; const currentWidth = canvas.width; const currentHeight = canvas.height; // Check if DPR is already applied (avoid double-scaling) const expectedWidth = Math.floor(window.innerWidth * dpr); const expectedHeight = Math.floor(window.innerHeight * dpr); if (Math.abs(currentWidth - expectedWidth) > 2 || Math.abs(currentHeight - expectedHeight) > 2) { canvas.width = expectedWidth; canvas.height = expectedHeight; console.debug("Applied DPR scaling for Needle AppClip XR session", dpr, canvas.width, canvas.height); } }); } } /** called when renderer.setSession is fulfilled */ private onRendererSessionSet = () => { if (!this.running) return; this.context.renderer.xr.enabled = true; // calling update camera here once to prevent the xr camera left/right NaN in the first frame due to false near/far plane values, see NE-4126 this.context.renderer.xr.updateCamera(this.context.mainCamera as PerspectiveCamera); this.context.mainCameraComponent?.applyClearFlags(); } private onInputSourceAdded = (newInputSource: XRInputSource) => { // do not create XR controllers for screen input sources if (newInputSource.targetRayMode === "screen") { return; } let index = 0; for (let i = 0; i < this.session.inputSources.length; i++) { if (this.session.inputSources[i] === newInputSource) { index = i; break; } } // check if an xr controller for this input source already exists // in case we have both an event from inputsourceschange and from the construtor initial input sources if (this.controllers.find(c => c.inputSource === newInputSource)) { console.debug("Controller already exists for input source", index); return; } else if (this._newControllers.find(c => c.inputSource === newInputSource)) { console.debug("Controller already registered for input source", index); return; } // TODO: check if this is a transient input source AND we can figure out which existing controller it likely belongs to // TODO: do not draw raycasts for controllers that don't have primary input actions / until we know that they have primary input actions const newController = new NeedleXRController(this, newInputSource, index); this._newControllers.push(newController); } /** Disconnects the controller, invokes events and notifies previou controller (if any) */ private disconnectInputSource(inputSource: XRInputSource) { const handleRemove = (oldController: NeedleXRController, array: Array) => { if (oldController.inputSource === inputSource) { if (debug) console.log("Disconnecting controller", oldController.index); const index = array.indexOf(oldController); if (index >= 0) array.splice(index, 1); this.invokeControllerEvent(oldController, this._controllerRemoved, "removed"); const args: NeedleXRControllerEventArgs = { xr: this, controller: oldController, change: "removed" }; for (const script of this._xr_scripts) { if (script.onXRControllerRemoved) script.onXRControllerRemoved(args); } oldController.onDisconnected(); } } const copy = [...this.controllers]; for (let i = copy.length - 1; i >= 0; i--) { const oldController = copy[i]; handleRemove(oldController, this.controllers); } const copyNew = [...this._newControllers]; for (let i = copyNew.length - 1; i >= 0; i--) { const oldController = copyNew[i]; handleRemove(oldController, this._newControllers); } } /** End the XR Session */ end() { // this can be called by external code to end the session // the actual cleanup happens in onEnd which subscribes to the session end event // so users can also just regularly call session.end() and the cleanup will happen automatically if (this._ended) return; this.session.end().catch(e => console.warn(e)); } private _ended: boolean = false; private readonly _newControllers: NeedleXRController[] = []; private onEnd = (_evt: XRSessionEvent) => { if (this._ended) return; this._ended = true; console.debug("XR Session ended"); Telemetry.sendEvent(Context.Current, "xr", { action: "session_end", mode: this.mode, source: "NeedleXRSession.onEnd", }); deleteSessionInfo(); this.onAfterRender(); this.revertCustomForward(); this._didStart = false; this._previousCameraParent = null; this.requestedCameraNearPlane = null; unregisterFrameEventCallback(this.onBefore, FrameEvent.LateUpdate); const index = this.context.pre_render_callbacks.indexOf(this.onBeforeRender); if (index >= 0) this.context.pre_render_callbacks.splice(index, 1); const index2 = this.context.post_render_callbacks.indexOf(this.onAfterRender); if (index2 >= 0) this.context.post_render_callbacks.splice(index2, 1); this.context.xr = null; this.context.renderer.xr.enabled = false; // apply the clearflags at the beginning of the next frame this.context.pre_update_oneshot_callbacks.push(() => { this.context.mainCameraComponent?.applyClearFlags() this.context.mainCameraComponent?.applyClippingPlane(); }); invokeXRSessionEnd({ session: this }); for (const listener of NeedleXRSession._xrEndListeners) { listener({ xr: this }); } // make sure we disconnect all controllers // we copy the array because the disconnectInputSource method modifies the controllers array const copy = [...this.controllers]; for (let i = 0; i < copy.length; i++) { this.disconnectInputSource(copy[i].inputSource); } this.controllers.length = 0; this._newControllers.length = 0; // we want to call leave XR for *all* scripts that are still registered // even if they might already be destroyed e.g. by the WebXR component (it destroys the default controller scripts) // they should still receive this callback to be properly cleaned up for (const listener of this._xr_scripts) { this._scripts_in_xr.delete(listener); listener?.onLeaveXR?.({ xr: this }); } this._scripts_in_xr.clear(); this.sync?.onExitXR(this); if (this.context.mainCamera) { // if we have a main camera we want to move it back to it's original parent this._originalCameraParent?.add(this.context.mainCamera); if (this._originalCameraWorldPosition) { setWorldPosition(this.context.mainCamera, this._originalCameraWorldPosition); } if (this._originalCameraWorldRotation) { setWorldQuaternion(this.context.mainCamera, this._originalCameraWorldRotation); } if (this._originalCameraWorldScale) { setWorldScale(this.context.mainCamera, this._originalCameraWorldScale); } if (this.context.mainCamera instanceof PerspectiveCamera) { if (this.context.mainCamera[$initialFov]) { this.context.mainCamera.fov = this.context.mainCamera[$initialFov]!; this.context.mainCamera[$initialFov] = 0; } if (this.context.mainCamera[$initialNear]) { this.context.mainCamera.near = this.context.mainCamera[$initialNear]!; this.context.mainCamera[$initialNear] = 0; } } } // mark for size change since DPI might have changed this.context.requestSizeUpdate(); this._defaultRig.gameObject.removeFromParent(); enableSpatialConsole(false); //performance.mark(measure_SessionEndedMarker); //performance.measure('NeedleXRSession', measure_SessionStartedMarker, measure_SessionEndedMarker); }; private _didStart: boolean = false; /** Called every frame by the engine */ private onBefore = (context: Context) => { const frame = context.xrFrame; if (!frame) return; //performance.mark('NeedleXRSession onBefore start'); // ensure that XR is always set to a running session this.context.xr = this; // ensure that we always have the correct main camera reference // we need to store the main camera reference here because the main camera can be disabled and then it's removed from the context // but in XR we want to ensure we always have a active camera (or at least parent the camera to the active rig) if (this.context.mainCameraComponent && this.context.mainCameraComponent !== this._mainCamera) { this._mainCamera = this.context.mainCameraComponent; } if (this.rig?.isActive == false) { if (debug) console.warn("Latest rig is not active - trying to activate a different rig", this.rig); this.updateActiveXRRig(); } // make sure the camera is parented to the active rig // Note: applyCustomForward() inserts _cameraRenderParent between rig and camera, // so the hierarchy is: rig → _cameraRenderParent → camera. We check the effective parent. if (this.rig) { const cameraObject = this._mainCamera?.gameObject ?? this.context.mainCamera; if (cameraObject) { const effectiveParent = cameraObject.parent === this._cameraRenderParent ? this._cameraRenderParent.parent : cameraObject.parent; if (effectiveParent !== this.rig.gameObject) { this.rig.gameObject.add(cameraObject); } } } this.internalUpdateState(); // we apply the flip immediately and keep it while in XR so that regular raycasts just work // otherwise rendering would fool us this.applyCustomForward(); const args: NeedleXREventArgs = { xr: this }; // we want to invoke ENTERXR for NEW component nad EXITXR for REMOVED components // we want to invoke onControllerAdded to components joining a running XR session if controllers are already connected //TODO: handle REMOVED components during a session (we want to call leave session events and controller removed events) // deferred start because we need an XR frame if (!this._didStart) { this._didStart = true; // place default rig to view the scene if (this.mode === "immersive-vr") { const bounds = getBoundingBox(this.context.scene.children); if (bounds) { const size = bounds.getSize(getTempVector()); if (size.length() > 0) { const rigobject = this._defaultRig.gameObject; rigobject.position.set(bounds.min.x + size.x * .5, bounds.min.y, bounds.max.z + size.z * .5 + 1.5); const centerLook = bounds.getCenter(getTempVector()); centerLook.y = rigobject.position.y; rigobject.lookAt(centerLook); } } } invokeXRSessionStart({ session: this }); internalOnUserInputRegistered(); for (const listener of NeedleXRSession._xrStartListeners) { listener(args); } // invoke session listeners start // we need to make a copy because the array might be modified during the loop (could also use a for loop and iterate backwards perhaps but then order of invocation would be changed OR check if the size has changed...) const copy = [...this._xr_scripts]; if (debug) console.log("NeedleXRSession start, handle scripts:", copy); for (const script of copy) { if (script.destroyed) { this._script_to_remove.push(script); continue; } if (!script.activeAndEnabled) { this.markInactive(script); continue; } // if ((script as IComponent).activeAndEnabled === false) continue; this.invokeCallback_EnterXR(script); // also invoke all events for currently (already) connected controllers for (const controller of this.controllers) { this.invokeCallback_ControllerAdded(script, controller); } } } else if (this.context.new_scripts_xr.length > 0) { // invoke start on all new scripts that were added during the session and that support the current mode const copy = [...this.context.new_scripts_xr]; for (let i = 0; i < copy.length; i++) { const script = this.context.new_scripts_xr[i] as IComponent & INeedleXRSessionEventReceiver; if (!script || script.destroyed || script.supportsXR?.(this.mode) == false) { this.context.new_scripts_xr.splice(i, 1); continue; } if (!script.activeAndEnabled) { this.context.new_scripts_xr.splice(i, 1); this.markInactive(script); continue; } // ignore inactive scripts // if (script.activeAndEnabled === false) continue; if (this.addScript(script)) { // invoke onEnterXR on those scripts because they joined a running session this.invokeCallback_EnterXR(script); // also invoke all events for currently (already) connected controllers for (const controller of this.controllers) { this.invokeCallback_ControllerAdded(script, controller); } } } } // make sure camera layers are correct // we do this every frame here but I think it would be enough to do it once after the first rendering // since we want to override the settings in three's WebXRManager // we also want to continuously sync the culling mask if a user changes it on the main cam while in XR this.syncCameraCullingMask(); // update controllers for (const controller of this.controllers) { controller.onUpdate(frame); } // handle when new controllers have been added if (this._newControllers.length > 0) { const copy = [...this._newControllers]; this._newControllers.length = 0; for (const controller of copy) { if (!controller.connected) { console.warn("New controller is not connected", controller); continue; } this.controllers.push(controller); for (const script of this._xr_scripts) { if (script.destroyed) { this._script_to_remove.push(script); continue; } if (script.activeAndEnabled === false) { continue; } this.invokeCallback_ControllerAdded(script, controller); } } this.controllers.sort((a, b) => a.index - b.index); } if (debug && this.context.time.frame % 30 === 0 && this.controllers.length <= 0 && this.session.inputSources.length > 0) { enableSpatialConsole(true) console.error("XRControllers are not added but inputSources are present"); } //performance.mark('NeedleXRSession update scripts start'); // check all XR scripts for destroyed or inactive state // this must cover _xr_scripts (not just _xr_update_scripts) so that scripts // without onUpdateXR are also detected as inactive when removed from the scene // iterate backwards since markInactive/removeScript modifies the array for (let i = this._xr_scripts.length - 1; i >= 0; i--) { const script = this._xr_scripts[i]; if (script.destroyed === true) { this._script_to_remove.push(script); continue; } if (script.activeAndEnabled === false) { this.markInactive(script); continue; } } // invoke update on scripts that have onUpdateXR for (const script of this._xr_update_scripts) { if (script.destroyed || script.activeAndEnabled === false) continue; if (script.onUpdateXR) script.onUpdateXR(args); } //performance.mark('NeedleXRSession update scripts end'); //performance.measure('NeedleXRSession update scripts', 'NeedleXRSession update scripts start', 'NeedleXRSession update scripts end'); // handle inactive scripts this.handleInactiveScripts(); // handle removed scripts if (this._script_to_remove.length > 0) { // make sure we have no duplicates const unique = [...new Set(this._script_to_remove)]; this._script_to_remove.length = 0; for (const script of unique) { if (!script.destroyed && this.running) { script.onLeaveXR?.(args); } this.removeScript(script); } } this.sync?.onUpdate(this); this.onRenderDebug(); //performance.mark('NeedleXRSession onBefore end'); //performance.measure('NE XR frame', 'NeedleXRSession onBefore start', 'NeedleXRSession onBefore end'); } private onRenderDebug() { if (debug) { for (const controller of this.controllers) { controller.onRenderDebug(); } } if ((debug || debugFPS) && this.rig) { debugFPSFramesSinceLastUpdate++; if (debugFPSFramesSinceLastUpdate >= 20) { const pos = this.rig.gameObject.worldPosition; const forward = this.rig.gameObject.worldForward; pos.add(forward.multiplyScalar(1.5)); const upwards = this.rig.gameObject.worldUp; pos.add(upwards.multiplyScalar(2.5)); let debugLabel = ""; debugLabel += `${this.context.time.smoothedFps.toFixed(0)} FPS`; debugLabel += `, calls: ${this.context.renderer.info.render.calls}, tris: ${this.context.renderer.info.render.triangles.toLocaleString()}`; if (debug || debugFPS) { for (const ctrl of this.controllers) { debugLabel += `\n${ctrl.hand ? "hand" : "ctrl"} ${ctrl.inputSource.handedness}[${ctrl.index}] con:${ctrl.connected} tr:${ctrl.isTracking} hts:${ctrl.hasHitTestSource ? "yes" : "no"}`; } } debugFPSFramesSinceLastUpdate = 0; Gizmos.DrawLabel(pos, debugLabel, undefined, 1 / 60 * 20); } } } private onBeforeRender = () => { if (this.context.mainCamera) { this.updateFade(this.context.mainCamera); if (this.requestedCameraNearPlane !== null && this.context.mainCamera instanceof PerspectiveCamera) { this.context.mainCamera.near = this.requestedCameraNearPlane; this.requestedCameraNearPlane = null; } } } private onAfterRender = () => { this.onUpdateFade_PostRender(); // render spectator view if we're in VR using Link // __rendered_once is for when we are on device, but opening the browser should not show a blank space if (DeviceUtilities.isDesktop() || !this["_renderOnceOnDevice"]) { const renderer = this.context.renderer; if (renderer.xr.isPresenting && this.context.mainCamera) { this["_renderOnceOnDevice"] = true; const wasXr = renderer.xr.enabled; const previousRenderTarget = renderer.getRenderTarget(); const previousBackground = this.context.scene.background; renderer.xr.enabled = false; renderer.setRenderTarget(null); if (this.isPassThrough) this.context.scene.background = null; if (this.context.composer) { this.context.composer.render(this.context.time.deltaTime); } else { renderer.render(this.context.scene, this.context.mainCamera); } renderer.xr.enabled = wasXr; renderer.setRenderTarget(previousRenderTarget); this.context.scene.background = previousBackground; } } } /** register a new XR script if it hasnt added yet */ private addScript(script: INeedleXRSessionEventReceiver) { if (this._xr_scripts.includes(script)) return false; if (debug) console.log("Register new XRScript", script); this._xr_scripts.push(script); if (typeof script.onUpdateXR === "function") { this._xr_update_scripts.push(script); } return true; } /** mark a script as inactive and invokes callbacks */ private markInactive(script: INeedleXRSessionEventReceiver) { if (this._inactive_scripts.indexOf(script) >= 0) return; // inactive scripts should not receive any regular callbacks anymore this.removeScript(script, false); this._inactive_scripts.push(script); // inactive scripts receive callbacks as if the XR session has ended for (const ctrl of this.controllers) this.invokeCallback_ControllerRemoved(script, ctrl); this.invokeCallback_LeaveXR(script); } private handleInactiveScripts() { if (this._inactive_scripts.length > 0) { for (let i = this._inactive_scripts.length - 1; i >= 0; i--) { const script = this._inactive_scripts[i]; if (script.activeAndEnabled) { this._inactive_scripts.splice(i, 1); // addScript returns false if already re-added (e.g. via new_scripts_xr processing) if (this.addScript(script)) { this.invokeCallback_EnterXR(script); for (const ctrl of this.controllers) this.invokeCallback_ControllerAdded(script, ctrl); } } } } } private readonly _script_to_remove: INeedleXRSessionEventReceiver[] = []; private removeScript(script: INeedleXRSessionEventReceiver, removeCompletely: boolean = true) { if (debug) console.log("Remove XRScript", script); const index = this._xr_scripts.indexOf(script); if (index >= 0) this._xr_scripts.splice(index, 1); const index2 = this._xr_update_scripts.indexOf(script); if (index2 >= 0) this._xr_update_scripts.splice(index2, 1); if (removeCompletely) { const index3 = this._inactive_scripts.indexOf(script); if (index3 >= 0) this._inactive_scripts.splice(index3, 1); } } private invokeCallback_EnterXR(script: INeedleXRSessionEventReceiver) { this._scripts_in_xr.add(script); if (script.onEnterXR) { script.onEnterXR({ xr: this }); } } private invokeCallback_ControllerAdded(script: INeedleXRSessionEventReceiver, controller: NeedleXRController) { if (script.onXRControllerAdded) { script.onXRControllerAdded({ xr: this, controller, change: "added" }); } } private invokeCallback_ControllerRemoved(script: INeedleXRSessionEventReceiver, controller: NeedleXRController) { if (script.onXRControllerRemoved) { script.onXRControllerRemoved({ xr: this, controller, change: "removed" }); } } private invokeCallback_LeaveXR(script: INeedleXRSessionEventReceiver) { if (!this._scripts_in_xr.delete(script)) return; if (script.onLeaveXR && !script.destroyed) { script.onLeaveXR({ xr: this }); } } private syncCameraCullingMask() { // when we set unity layers objects will only be rendered on one eye // we set layers to sync raycasting and have a similar behaviour to unity const cam = this.context.xrCamera; const cull = this.context.mainCameraComponent?.cullingMask; if (cam && cull !== undefined) { for (const c of cam.cameras) { c.layers.mask = cull; } cam.layers.mask = cull; } else if (cam) { for (const c of cam.cameras) { c.layers.enableAll(); } cam.layers.enableAll(); } } private invokeControllerEvent(controller: NeedleXRController, listeners: ControllerChangedEvt[], change: "added" | "removed") { for (let i = listeners.length - 1; i >= 0; i--) { const listener = listeners[i]; if (!listener) continue; try { listener({ xr: this, controller, change }); } catch (e) { console.error(e); } } } private _camera!: Object3D; private readonly _cameraRenderParent: Object3D = new Object3D().rotateY(Math.PI); private _previousCameraParent!: Object3D | null; private readonly _customforward: boolean = true; private originalCameraNearPlane?: number; private requestedCameraNearPlane: number | null = null; /** This is used to have the XR system camera look into threejs Z forward direction (instead of -z) */ private applyCustomForward() { if (this.context.mainCamera && this._customforward) { this._camera = this.context.mainCamera; if (this._camera.parent !== this._cameraRenderParent) { this._previousCameraParent = this._camera.parent; this._previousCameraParent?.add(this._cameraRenderParent); } this._cameraRenderParent.name = "XR Camera Render Parent"; this._cameraRenderParent.add(this._camera); { let minNearPlane = .02; const threshold = 0.001; if (this.rig) { const rigWorldScale = getWorldScale(this.rig.gameObject); minNearPlane *= rigWorldScale.x; } if (this._camera instanceof PerspectiveCamera && Math.abs(this._camera.near - minNearPlane) > threshold) { // TODO: when we're in AR and update the near plane we do get glitchy rendering/sheered matrix transforms when scaling the scene (WebXRSessionRoot). As a workaround we defer AR near plane changes to before rendering but change VR near planes immediately. if (this.isAR) this.originalCameraNearPlane = this._camera.near; else { this._camera.near = minNearPlane; } if (debug) console.debug(`Setting camera near plane to ${minNearPlane} (was ${this.originalCameraNearPlane}) to account for XR rendering scale`); } } } } private revertCustomForward() { if (this._camera && this._previousCameraParent) { this._previousCameraParent.add(this._camera); } this._previousCameraParent = null; if (this._camera instanceof PerspectiveCamera && this.originalCameraNearPlane != undefined) { this._camera.near = this.originalCameraNearPlane; this.originalCameraNearPlane = undefined; } } private _viewerPose?: XRViewerPose; private readonly _transformOrientation = new Quaternion(); private readonly _transformPosition = new Vector3(); private internalUpdateState() { const referenceSpace = this.context.renderer.xr.getReferenceSpace(); if (!referenceSpace) { this._viewerPose = undefined; return; } this._viewerPose = this.frame.getViewerPose(referenceSpace); if (this._viewerPose) { const transform: XRRigidTransform = this._viewerPose.transform; this._transformPosition.set(transform.position.x, transform.position.y, transform.position.z); this._transformOrientation.set(transform.orientation.x, transform.orientation.y, transform.orientation.z, transform.orientation.w); } } // TODO: for scene transitions (e.g. SceneSwitcher) where creating the scene might take a few moments we might want more control over when/how this fading occurs and how long the scene stays black private _transition?: SceneTransition; public get transition() { if (!this._transition) this._transition = new SceneTransition(); return this._transition; } /** Call to fade rendering to black for a short moment (the returned promise will be resolved when fully black) * This can be used to mask scene transitions or teleportation * @returns a promise that is resolved when the screen is fully black * @example `fadeTransition().then(() => { })` */ fadeTransition() { if (!this._transition) this._transition = new SceneTransition(); return this._transition.fadeTransition(); } /** e.g. FadeToBlack */ private updateFade(camera: Camera) { if (this._transition && camera instanceof PerspectiveCamera) this._transition.update(camera, this.context.time.deltaTime); } private onUpdateFade_PostRender() { this._transition?.remove(); } }