import { ImageBitmapLoader, Matrix4, Object3D, Quaternion, Vector3 } from "three"; import { Object3DEventMap } from "three"; import { isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../../engine/debug/index.js"; import { AssetReference } from "../../engine/engine_addressables.js"; import { Context } from "../../engine/engine_context.js"; import { Telemetry } from "../../engine/engine_license.js"; import { serializable } from "../../engine/engine_serialization.js"; import { IGameObject } from "../../engine/engine_types.js"; import { CircularBuffer, delay, DeviceUtilities, getParam } from "../../engine/engine_utils.js"; import { type NeedleXREventArgs, NeedleXRSession } from "../../engine/xr/api.js"; import { IUSDExporterExtension } from "../../engine-components/export/usdz/Extension.js"; import { imageToCanvas, USDObject, USDWriter, USDZExporterContext } from "../../engine-components/export/usdz/ThreeUSDZExporter.js"; import { USDZExporter } from "../../engine-components/export/usdz/USDZExporter.js"; import { Behaviour, GameObject } from "../Component.js"; import { EventList } from "../EventList.js"; import { Renderer } from "../Renderer.js"; import { _imageElements, collectTrackedImages, loadImage } from "./WebXRImageTracking.utils.js"; // https://github.com/immersive-web/marker-tracking/blob/main/explainer.md const debug = getParam("debugimagetracking"); // #region WebXRTrackedImage /** * Represents a tracked image detected during a WebXR session. * Contains position, rotation, and tracking state information for a detected marker image. * * **Properties:** * - Access image URL and physical dimensions * - Get current position and rotation in world space * - Check tracking state (tracked vs emulated) * - Apply transform to 3D objects * * @summary Runtime data for a detected marker image in WebXR * @category XR */ export class WebXRTrackedImage { /** URL of the tracked marker image */ get url(): string { return this._trackedImage.image ?? ""; } /** Physical width of the marker in meters */ get widthInMeters() { return this._trackedImage.widthInMeters ?? undefined; } /** The ImageBitmap used for tracking */ get bitmap(): ImageBitmap { return this._bitmap; } /** * The {@link WebXRImageTrackingModel} configuration for this tracked image. * Use this to access the assigned 3D object, marker settings, and other image tracking configuration. * Available on each {@link WebXRTrackedImage} received from the `image-tracking` {@link CustomEvent} (`event.detail`). * @example * ```ts * tracker.addEventListener("image-tracking", event => { * for (const img of event.detail.trackedImages) { * const model = img.model; * // Access the assigned 3D object * const obj = model.object; * // Access other settings * console.log(model.widthInMeters, model.hideWhenTrackingIsLost); * } * }); * ``` */ get model(): WebXRImageTrackingModel { return this._trackedImage; } /** * The 3D object or prefab assigned to this tracked image marker in the {@link WebXRImageTrackingModel}. * Use this to access the object associated with an AR image tracking marker from the `image-tracking` {@link CustomEvent}. * Shorthand for `this.model.object`. * @example * ```ts * tracker.addEventListener("image-tracking", event => { * for (const img of event.detail.trackedImages) { * const obj = img.trackedModel; * // verbose alternative: img.model.object * } * }); * ``` */ get trackedModel(): AssetReference | undefined { return this._trackedImage.object; } /** * The measured size of the detected image in the real world. * May differ from `widthInMeters` if the physical marker doesn't match the configured size. */ readonly measuredSize: number; /** * Current tracking state of the image: * - `tracked` - Image is currently being tracked by the system * - `emulated` - Tracking is being emulated (less accurate) */ readonly state: "tracked" | "emulated"; /** * Copy the current world position of the tracked image to a Vector3. * @param vec The vector to store the position in * @returns The input vector with the position copied to it */ getPosition(vec: Vector3) { this.ensureTransformData(); vec.copy(this._position); return vec; } /** * Copy the current world rotation of the tracked image to a Quaternion. * @param quat The quaternion to store the rotation in * @returns The input quaternion with the rotation copied to it */ getQuaternion(quat: Quaternion) { this.ensureTransformData(); quat.copy(this._rotation); return quat; } /** * Apply the tracked image's position and rotation to a 3D object. * Optionally applies smoothing to reduce jitter. * * @param object The 3D object to update * @param t01 Interpolation factor (0-1) for smoothing. If undefined or >= 1, no smoothing is applied. When smoothing is enabled, larger position/rotation changes will automatically reduce the smoothing to prevent lag. */ applyToObject(object: Object3D, t01: number | undefined = undefined) { this.ensureTransformData(); // check if position/_position or rotation/_rotation changed more than just a little bit and adjust smoothing accordingly const changeAmount = object.position.distanceToSquared(this._position) / 0.05 + object.quaternion.angleTo(this._rotation) / 0.05; if (t01) t01 *= Math.max(1, changeAmount); if (t01 === undefined || t01 >= 1) { object.position.copy(this._position); object.quaternion.copy(this._rotation); // InstancingUtil.markDirty(object); } else { t01 = Math.max(0, Math.min(1, t01)); object.position.lerp(this._position, t01); object.quaternion.slerp(this._rotation, t01); // InstancingUtil.markDirty(object); } } private static _positionBuffer: CircularBuffer = new CircularBuffer(() => new Vector3(), 20); private static _rotationBuffer: CircularBuffer = new CircularBuffer(() => new Quaternion(), 20); private _position!: Vector3; private _rotation!: Quaternion; private ensureTransformData() { if (!this._position) { this._position = WebXRTrackedImage._positionBuffer.get(); this._rotation = WebXRTrackedImage._rotationBuffer.get(); const t = this._pose.transform as XRRigidTransform; const converted = NeedleXRSession.active!.convertSpace(t); this._position.copy(converted?.position); this._rotation.copy(converted?.quaternion); } } private readonly _trackingComponent: WebXRImageTracking; private readonly _trackedImage: WebXRImageTrackingModel; private readonly _bitmap: ImageBitmap; private readonly _pose: any; constructor(context: WebXRImageTracking, trackedImage: WebXRImageTrackingModel, bitmap: ImageBitmap, measuredSize: number, state: "tracked" | "emulated", pose: any) { this._trackingComponent = context;; this._trackedImage = trackedImage; this._bitmap = bitmap; this.measuredSize = measuredSize; this.state = state; this._pose = pose; } } /** * Initial state of tracked image objects before entering an XR session. * Used to restore objects to their original state when the WebXR session ends. */ declare type InitialTrackedObjectState = { /** Original visibility state */ visible: boolean; /** Original parent object in the scene hierarchy */ parent: Object3D | undefined | null; /** Original transformation matrix */ matrix: Matrix4; } // #region Model /** * Configuration model for a tracked image marker. * Defines which image to track, its physical size, and which 3D content to display when detected. * * **Important:** The physical size (`widthInMeters`) must match your printed marker size for accurate tracking. * Mismatched sizes cause the tracked object to appear to "float" above or below the marker. * * **Best practices for marker images:** * - Use high-contrast images with distinct features * - Avoid repetitive patterns or solid colors * - Test images at intended viewing distances * - Ensure good lighting conditions * * @summary Configuration for a single trackable image marker * @category XR * @see {@link WebXRImageTracking} for the component that uses these models * @link https://engine.needle.tools/docs/xr.html#image-tracking * @link https://engine.needle.tools/samples/image-tracking */ export class WebXRImageTrackingModel { /** * Creates a new image tracking configuration. * * @param params Configuration parameters * @param params.url URL to the marker image to track * @param params.widthInMeters Physical width of the printed marker in meters (must match real size!) * @param params.object The 3D object or AssetReference to display when this image is detected * @param params.createObjectInstance If true, creates a new instance for each detection (useful for tracking multiple instances of the same marker) * @param params.imageDoesNotMove Enable for static markers (floor/wall mounted) to improve tracking stability * @param params.hideWhenTrackingIsLost If true, hides the object when tracking is lost; if false, leaves it at the last known position */ constructor(params: Omit) { this.image = params.url; if (params.widthInMeters !== undefined) this.widthInMeters = params.widthInMeters; if (params.object instanceof Object3D) { this.object = new AssetReference({ asset: params.object }); } else this.object = params.object; if (params.createObjectInstance !== undefined) this.createObjectInstance = params.createObjectInstance; if (params.imageDoesNotMove !== undefined) this.imageDoesNotMove = params.imageDoesNotMove; if (params.hideWhenTrackingIsLost !== undefined) this.hideWhenTrackingIsLost = params.hideWhenTrackingIsLost; } /** * URL to the marker image to track. * **Important:** Use images with high contrast and unique features to improve tracking quality. * Avoid repetitive patterns, solid colors, or low-contrast images. */ @serializable(URL) image?: string; /** * Physical width of the printed marker in meters. * **Critical:** This must match your actual printed marker size! * If mismatched, the tracked object will appear to "float" above or below the marker. * * @default 0.25 (25cm) * @example * ```ts * // For a business card sized marker (9cm wide) * widthInMeters = 0.09; * * // For an A4 page width (21cm) * widthInMeters = 0.21; * ``` */ @serializable() widthInMeters: number = .25; /** * The 3D object or prefab to display when this marker is detected. * The object will be positioned and rotated to match the tracked image in the real world. * * **Note:** Scale your 3D content appropriately relative to `widthInMeters`. */ @serializable(AssetReference) object?: AssetReference; /** * When enabled, creates a new instance of the referenced object each time this image is detected. * Enable this if you want to track multiple instances of the same marker simultaneously, * or if the same object is used for multiple different markers. * * @default false */ @serializable() createObjectInstance: boolean = false; /** * Enable for static markers that don't move (e.g., posters on walls or markers on the floor). * When enabled, only the first few tracking frames are used to position the object, * resulting in more stable tracking by ignoring subsequent minor position changes. * * **Use cases:** * - Wall-mounted posters or artwork * - Floor markers for persistent AR content * - Product packaging on shelves * * **Don't use for:** * - Handheld cards or objects * - Moving markers * * @default false */ @serializable() imageDoesNotMove: boolean = false; /** * Controls visibility behavior when tracking is lost. * - When `true`: Object is hidden when the marker is no longer visible * - When `false`: Object remains visible at its last tracked position * * @default true */ @serializable() hideWhenTrackingIsLost: boolean = true; /** * Extracts the filename from the marker image URL. * @returns The filename (last part of the URL path), or null if no image URL is set * @example * ```ts * // URL: "https://example.com/markers/business-card.png" * // Returns: "business-card.png" * ``` */ getNameFromUrl() { if (this.image) { const parts = this.image.split("/"); return parts[parts.length - 1]; } return null; } } /** * Options for adding a trackable image marker via {@link WebXRImageTracking.addImage}. * A convenience shape that also accepts a plain {@link Object3D} for `object`. */ export interface WebXRImageTrackingOptions { /** URL to the marker image to track. Use high-contrast images with unique features for reliable tracking. */ url?: string; /** The 3D object (or {@link AssetReference}) to display when the marker is detected. */ object: AssetReference | Object3D; /** Physical width of the printed marker in meters. Must match the real marker size. @default 0.25 */ widthInMeters?: number; /** Create a new instance of the object for each detection. @default false */ createObjectInstance?: boolean; /** Enable for static markers (walls/floor) to improve tracking stability. @default false */ imageDoesNotMove?: boolean; /** Hide the object when tracking is lost. @default true */ hideWhenTrackingIsLost?: boolean; /** Make this the primary marker (used in QuickLook fallback mode). @default false */ asPrimary?: boolean; } /** Data passed to image tracking event listeners. */ export interface WebXRImageTrackingEvent { /** The images currently being tracked this frame. */ readonly trackedImages: readonly WebXRTrackedImage[]; } /** Event map for {@link WebXRImageTracking} events. Use with `addEventListener` for typed event handling. */ export interface WebXRImageTrackingEventMap { /** Dispatched every frame when images are being tracked. The event detail contains the tracking data for the current frame. */ "image-tracking": CustomEvent; } // #region USDZ Extension class ImageTrackingExtension implements IUSDExporterExtension { readonly isImageTrackingExtension = true; get extensionName() { return "image-tracking"; } constructor(private readonly exporter: USDZExporter, private readonly component: WebXRImageTracking) { if (debug) console.log(this); this.exporter.anchoringType = "image"; } // set during export private shouldExport: boolean = true; private filename: string | null = null; private imageModel: WebXRImageTrackingModel | null = null; onBeforeBuildDocument(_context: USDZExporterContext) { // check if this extension is the first image tracking extension in the list // since iOS can only track one image at a time we only allow one image tracking extension to be active // we have to determine this at the earlierst export callback // all subsequent export callbacks should then check is shouldExport is set to true // this should only be the case for exactly one extension const index = this.exporter.extensions .filter(e => { const ext = (e as ImageTrackingExtension); return ext.isImageTrackingExtension && ext.component.activeAndEnabled && ext.component.trackedImages?.length > 0; }) .indexOf(this); this.shouldExport = index === 0; if (!this.shouldExport) return; // Warn if more than one tracked image is used for USDZ; that's not supported at the moment. if (this.component.trackedImages?.length > 1) { if (debug || isDevEnvironment()) { showBalloonWarning("USDZ: Only one tracked image is supported."); console.warn("USDZ: Only one tracked image is supported. Will choose the first one in the trackedImages list"); } } } onAfterHierarchy(_context: USDZExporterContext, writer: USDWriter) { if (!this.shouldExport) return; const iOSVersion = DeviceUtilities.getiOSVersion(); const majorVersion = iOSVersion ? parseInt(iOSVersion.split(".")[0]) : 18; const workaroundForFB16119331 = majorVersion >= 18; const multiplier = workaroundForFB16119331 ? 1 : 100; writer.beginBlock(`def Preliminary_ReferenceImage "AnchoringReferenceImage"`); writer.appendLine(`uniform asset image = @image_tracking/` + this.filename + `@`); writer.appendLine(`uniform double physicalWidth = ` + (this.imageModel!.widthInMeters * multiplier).toFixed(8)); writer.closeBlock(); } async onAfterSerialize(context: USDZExporterContext) { if (!this.shouldExport) return; const imageModel = this.imageModel; const img = _imageElements.get(imageModel!.image!)!; const canvas = await imageToCanvas(img); const blob = await canvas.convertToBlob({ type: 'image/png' }); const arrayBuffer = await blob.arrayBuffer(); context.files['image_tracking/' + this.filename] = new Uint8Array(arrayBuffer); } onExportObject(object: Object3D, model: USDObject, _context: USDZExporterContext) { if (!this.shouldExport) return; const imageTracking = this.component; if (!imageTracking || !imageTracking.trackedImages?.length || !imageTracking.activeAndEnabled) return; // we only care about the first image // We can only apply this to the first tracked image, more are not supported by QuickLook. const trackedImage = imageTracking.trackedImages[0]; if (trackedImage.object?.asset === object) { this.imageModel = trackedImage; this.filename = trackedImage.getNameFromUrl() || "marker.png"; const { scale, target } = this.exporter.getARScaleAndTarget(); // We have to reset the image tracking object's position and rotation, because QuickLook applies them. // On Android WebXR they're replaced by the tracked data let parent = object; const relativeMatrix = new Matrix4(); if (object !== target) { while (parent && parent.parent && parent.parent !== target) { parent = parent.parent; relativeMatrix.premultiply(parent.matrix); } } const mat = relativeMatrix .clone() .invert() // apply session root scale again after undoing the world transformation model.setMatrix(mat.scale(new Vector3(scale, scale, scale))); // Unfortunately looks like Apple's docs are incomplete: // https://developer.apple.com/documentation/realitykit/preliminary_anchoringapi#Nest-and-Layer-Anchorable-Prims // In practice, it seems that nesting is not allowed – no image tracking will be applied to nested objects. // Thus, we can't have separate transforms for "regularly placing content" and "placing content with an image marker". // model.extraSchemas.push("Preliminary_AnchoringAPI"); // model.addEventListener("serialize", (_writer: USDWriter, _context: USDZExporterContext) => { // writer.appendLine( `token preliminary:anchoring:type = "image"` ); // writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = ` ); // }); } } } // #region Tracking Component /** * Create powerful AR image tracking experiences with just a few lines of code! * WebXRImageTracking makes it incredibly easy to detect marker images in the real world and anchor 3D content to them. * Needle Engine automatically handles all the complexity across different platforms and fallback modes for you. * * [![Image Tracking Demo](https://cloud.needle.tools/-/media/vRUf9BmqW_bgNARATjmfCQ.gif)](https://engine.needle.tools/samples/image-tracking) * * **What makes Needle Engine special:** * - **Write once, run everywhere**: The same code works across iOS, Android, and visionOS * - **Automatic platform optimization**: Seamlessly switches between WebXR, ARKit, and QuickLook * - **Flexible deployment options**: From full WebXR with unlimited markers to QuickLook fallback * - **Production ready**: Battle-tested tracking with adaptive smoothing and stability features * * **Platform Support & Options:** * - **iOS (WebXR via AppClip)**: Full WebXR support - track unlimited markers simultaneously via native ARKit! * - **iOS (QuickLook mode)**: Instant AR without app installation - perfect for quick demos (tracks first marker) * - **Android (WebXR)**: Native WebXR Image Tracking API - unlimited markers (requires browser flag during early access) * - **visionOS (QuickLook)**: Spatial image anchoring with Apple's AR QuickLook * * **Simple 3-Step Setup:** * 1. Add this component to any GameObject in your scene * 2. Configure your markers in the `trackedImages` array: * - `image`: URL to your marker image * - `widthInMeters`: Physical size of your printed marker * - `object`: The 3D content to display * 3. Export and test - Needle handles the rest! * * **Pro Tips for Best Results:** * - Use high-contrast markers with unique features for reliable tracking * - Match `widthInMeters` to your actual physical marker size for accurate positioning * - Enable `imageDoesNotMove` for wall posters or floor markers - significantly improves stability * - Use `smooth` (enabled by default) for professional-looking, jitter-free tracking * - Test with different marker sizes and lighting - Needle's adaptive tracking handles various conditions * * ![](https://cloud.needle.tools/-/media/V-2UxGVRJxvH9oDnXGnIdg.png) * *WebXRImageTracking component in Unity Editor* * * ![](https://cloud.needle.tools/-/media/poDPca1bI1an4SBY7LtKNA.png) * *WebXRImageTracking panel/component in Blender* * * @example Getting started - it's this easy! * ```ts * // Just add markers and Needle handles everything else * const imageTracking = myObject.addComponent(WebXRImageTracking); * const marker = new WebXRImageTrackingModel({ * url: "https://example.com/my-poster.png", * widthInMeters: 0.3, // 30cm poster * object: my3DContent * }); * imageTracking.addImage(marker); * // Done! Works on iOS, Android, and visionOS automatically * ``` * * @example Track multiple markers (WebXR mode) * ```ts * const imageTracking = myObject.addComponent(WebXRImageTracking); * * // In WebXR mode (iOS AppClip, Android), all markers work simultaneously! * const productBox = new WebXRImageTrackingModel({ url: "product-box.png", widthInMeters: 0.15, object: productInfo }); * const businessCard = new WebXRImageTrackingModel({ url: "business-card.png", widthInMeters: 0.09, object: contactCard }); * const poster = new WebXRImageTrackingModel({ url: "poster.png", widthInMeters: 0.5, object: videoPlayer }); * * imageTracking.addImage(productBox); * imageTracking.addImage(businessCard); * imageTracking.addImage(poster); * * // For QuickLook fallback mode, optionally set which marker is primary * imageTracking.setPrimaryImage(poster); // This will be used in QuickLook * ``` * * @example Professional setup for static markers * ```ts * // Perfect for museums, retail displays, or permanent installations * const wallArt = new WebXRImageTrackingModel({ * url: "gallery-painting.png", * widthInMeters: 0.6, * object: interactiveExplanation, * imageDoesNotMove: true, // Rock-solid tracking for static markers! * hideWhenTrackingIsLost: false // Content stays visible even if temporarily occluded * }); * ``` * * **Why developers love Needle's image tracking:** * - Zero platform-specific code required * - Automatic graceful degradation across deployment modes * - Built-in jitter reduction and stability features * - Works with any image - posters, packaging, business cards, artwork * - Export once, deploy everywhere * * @summary The easiest way to create cross-platform AR image tracking experiences * @category XR * @group Components * @see {@link WebXRImageTrackingModel} for marker configuration options * @see {@link WebXRTrackedImage} for runtime tracking data and events * @see {@link WebXR} for general WebXR setup and session management * @link https://engine.needle.tools/docs/xr.html#image-tracking - Full Documentation * @link https://engine.needle.tools/samples/image-tracking - Try Live Demo * @link https://github.com/immersive-web/marker-tracking/blob/main/explainer.md - WebXR Marker Tracking Specification */ export class WebXRImageTracking extends Behaviour { /** * Event invoked every frame when images are being tracked. * @example * ```ts * const tracker = this.gameObject.getComponent(WebXRImageTracking); * tracker?.imageTracked.addEventListener(evt => { * for (const img of evt.trackedImages) { * console.log(img.url, img.state); * } * }); * ``` */ @serializable(EventList) imageTracked: EventList = new EventList(); /** @inheritdoc */ addEventListener(type: K, listener: (evt: WebXRImageTrackingEventMap[K]) => any): void; addEventListener(type: string, listener: (evt: T) => any): void; addEventListener(type: string, listener: (evt: any) => any): void { super.addEventListener(type, listener); } /** @inheritdoc */ removeEventListener(type: K, listener: (evt: WebXRImageTrackingEventMap[K]) => any): void; removeEventListener(type: string, listener: (evt: T) => any): void; removeEventListener(type: string, listener: (evt: any) => any): void { super.removeEventListener(type, listener); } /** * Set which marker should be primary (first in the list). * Useful when deploying to QuickLook mode where one marker is tracked at a time. * In full WebXR mode (iOS AppClip, Android), all markers track simultaneously regardless of order. * * **Note:** Needle Engine automatically adapts - in WebXR all markers work, in QuickLook the primary is used. * * @param image The marker model to set as primary * * @example * ```ts * // Great for offering different AR experiences from one deployment * imageTracking.setPrimaryImage(businessCardMarker); // Use this for QuickLook * // In WebXR mode, all markers still work simultaneously! * ``` */ setPrimaryImage(image: WebXRImageTrackingModel) { const index = this.trackedImages.indexOf(image); if (index >= 0) { const current = this.trackedImages[0]; if (current !== image) { this.trackedImages[0] = image; this.trackedImages[index] = current; } } else console.warn(`[WebXRImageTracking] Can not set primary: image not found in 'trackedImages' array ${image.image}`); } /** * Add a marker to track - it's that simple! * Needle Engine handles all the platform differences automatically. * * **Tip:** Add all your markers upfront. In WebXR mode they all work simultaneously. * In QuickLook mode, the first (primary) marker is used. * * @param image The marker configuration - either a plain options object or a {@link WebXRImageTrackingModel}. * Set `asPrimary: true` to make this the primary marker (for QuickLook fallback). * * @example * ```ts * // Super simple - just pass the marker url and the object to show * imageTracking.addImage({ * url: "https://mysite.com/poster.png", * object: cool3DModel, * widthInMeters: 0.42, // A3 poster width (optional, defaults to 0.25) * }); * // That's it! Needle does the rest. * ``` */ addImage(options: WebXRImageTrackingOptions): void; addImage(model: WebXRImageTrackingModel): void; addImage(image: WebXRImageTrackingOptions | WebXRImageTrackingModel): void { // Accept either a ready-made model or a plain options object (which allows `object` to be a // plain Object3D). Normalize to a WebXRImageTrackingModel; the constructor wraps Object3D in // an AssetReference for us. const model = image instanceof WebXRImageTrackingModel ? image : new WebXRImageTrackingModel(image); if (!this.trackedImages.includes(model)) { this.trackedImages.push(model); loadImage(model.image!); if(isDevEnvironment()) { if(model.widthInMeters <= 0) { this.onInvalidImageSizeDetected(model.image!); } } } if (!(image instanceof WebXRImageTrackingModel) && image.asPrimary) this.setPrimaryImage(model); } /** * Your list of markers to track. Add as many as you need! * * **How it works across platforms:** * - **WebXR mode** (iOS AppClip, Android): All markers are tracked simultaneously - amazing for multi-marker experiences! * - **QuickLook mode** (iOS fallback, visionOS): First marker is used - perfect for quick demos without app installation * * **Needle's smart deployment:** Configure all your markers once, and Needle automatically uses the best * tracking mode available on each platform. No platform-specific code needed! * * @see {@link WebXRImageTrackingModel} for marker configuration * @see {@link addImage} and {@link setPrimaryImage} for runtime management */ @serializable(WebXRImageTrackingModel) readonly trackedImages: WebXRImageTrackingModel[] = []; /** * Enable Needle's professional-grade adaptive smoothing for rock-solid tracking. * Automatically reduces jitter while staying responsive to real movement. * * **Pro tip:** Keep this enabled (default) for production experiences! * * @default true */ @serializable() smooth: boolean = true; private readonly trackedImageIndexMap: Map = new Map(); /** * Check if image tracking is available on this device right now. * * **Note:** On Android Chrome, WebXR Image Tracking is currently behind a flag during the early access period. * Needle automatically falls back to other modes when needed, so your experience keeps working! */ get supported() { return this._supported; } private _supported: boolean = true; /** @internal */ awake(): void { if (debug) console.log(this) if (!this.trackedImages) return; for (const trackedImage of this.trackedImages) { if (trackedImage.image) { loadImage(trackedImage.image); if(isDevEnvironment() && trackedImage.widthInMeters <= 0) { this.onInvalidImageSizeDetected(trackedImage.image); } } } } /** @internal */ onEnable() { USDZExporter.beforeExport.addEventListener(this.onBeforeUSDZExport); } /** @internal */ onDisable(): void { USDZExporter.beforeExport.removeEventListener(this.onBeforeUSDZExport); } private onInvalidImageSizeDetected(url:string) { if(globalThis["__NEEDLE_DEBUG_WebARImageSizeInvalid__"]) return; globalThis["__NEEDLE_DEBUG_WebARImageSizeInvalid__"] = true; console.warn(`[WebXRImageTracking] Invalid marker size detected for image ${url}. Ensure 'widthInMeters' is greater than 0 for accurate tracking.`); if(isDevEnvironment()) { showBalloonWarning(`[WebXRImageTracking] Invalid marker size detected for image ${url}. Ensure 'widthInMeters' is greater than 0 for accurate tracking.`); } } private onBeforeUSDZExport = (args: { exporter: USDZExporter }) => { if (this.activeAndEnabled && this.trackedImages?.length) { args.exporter.extensions.push(new ImageTrackingExtension(args.exporter, this)); } } /** @internal */ async onBeforeXR(_mode: XRSessionMode, args: XRSessionInit & { trackedImages: Array }): Promise { if (!this.trackedImages) return; args.optionalFeatures = args.optionalFeatures || []; if (!args.optionalFeatures.includes("image-tracking")) args.optionalFeatures.push("image-tracking"); // Resolve the marker bitmaps *before* the session is requested. // onBeforeXR runs the moment "Enter AR" is tapped. Marker images load asynchronously // (network fetch + createImageBitmap), so without awaiting here a marker that hasn't finished // decoding yet would be silently dropped from `trackedImages` and never registered with the // native/WebXR session - it would then only start working after exiting and re-entering AR, // once the bitmap has been cached. Awaiting the (idempotent) loads closes that race. // A marker that can not be loaded resolves to null and is skipped individually, so a single // unavailable image never rejects the whole XR session request. See forum topic #2859. this.trackedImageIndexMap.clear(); const indexMap = await collectTrackedImages(this.trackedImages, args); for (const [index, model] of indexMap) { this.trackedImageIndexMap.set(index, model); } } /** @internal */ onEnterXR(args: NeedleXREventArgs): void { if (this.trackedImages) { for (const trackedImage of this.trackedImages) { if (trackedImage.object?.asset) { // capture the initial state of tracked images in the scene to restore them when the session ends const obj = trackedImage.object.asset as Object3D; if (!obj.userData) obj.userData = {}; const state: InitialTrackedObjectState = { visible: obj.visible, parent: obj.parent, matrix: obj.matrix.clone() }; obj.userData["image-tracking"] = state; } } } // clear out all frame counters for tracking for (const trackedData of this.imageToObjectMap.values()) { trackedData.frames = 0; } // A session GRANTED before the scene finished loading (App Clip / NeedleGo // handoff, Quest link traversal) was requested without this component's // markers — onBeforeXR never contributed to its init, and the WebXR spec has // no way to add tracked images to a running session. The NeedleGo polyfill // CAN (ARKit re-runs its configuration with new detection images), so // register late where supported and surface the misconfiguration otherwise. if (this.trackedImages?.length && this.trackedImageIndexMap.size === 0) { if (isDevEnvironment() || debug) console.debug(`[WebXRImageTracking] No marker images made it into the session init (granted session?) — attempting runtime registration for ${this.trackedImages.length} image(s)`); this.registerImagesWithRunningSession(args.xr.session); } else if (this.trackedImages?.length && (isDevEnvironment() || debug)) { console.debug(`[WebXRImageTracking] ${this.trackedImageIndexMap.size} marker image(s) registered via the session init`); } }; /** Late-registers this component's marker images into an already running session * via the NeedleGo polyfill's runtime hook. See {@link onEnterXR}. */ private async registerImagesWithRunningSession(session: XRSession) { const register = (session as XRSession & { nonStandard_registerTrackedImages?: (images: Array<{ image: ImageBitmap, widthInMeters: number }>) => number }).nonStandard_registerTrackedImages; if (typeof register !== "function") { console.warn("[WebXRImageTracking] The XR session was started before the scene finished loading and does not support registering marker images at runtime — image tracking will not work in this session. Exit and re-enter AR from the loaded page."); Telemetry.sendEvent(this.context, "xr", { action: "image_tracking_missed_session_init" }); return; } const args: { trackedImages?: Array<{ image: ImageBitmap, widthInMeters: number }> } = {}; const indexMap = await collectTrackedImages(this.trackedImages, args); if (!args.trackedImages?.length) return; // the session may have ended (or been replaced) while marker bitmaps loaded if (this.context.xr?.session !== session) return; const baseIndex = register.call(session, args.trackedImages); if (baseIndex < 0) return; for (const [index, model] of indexMap) { this.trackedImageIndexMap.set(baseIndex + index, model); } if (isDevEnvironment() || debug) console.debug(`[WebXRImageTracking] Registered ${args.trackedImages.length} marker image(s) with the running session (granted before the scene loaded)`); } /** @internal */ onLeaveXR(_args: NeedleXREventArgs): void { // result indices are scoped to the session that registered the images — a new // session rebuilds the map (onBeforeXR, or the late registration in onEnterXR // for granted sessions, which detects the empty map) this.trackedImageIndexMap.clear(); if (!this.supported && DeviceUtilities.isAndroidDevice()) { showBalloonWarning(this.webXRIncubationsWarning); } if (this.trackedImages) { for (const trackedImage of this.trackedImages) { if (trackedImage.object?.asset) { const obj = trackedImage.object.asset as Object3D; if (obj.userData) { // restore the initial state of tracked images in the scene const state = obj.userData["image-tracking"] as InitialTrackedObjectState | undefined; if (state) { obj.visible = state.visible; state.parent?.add(obj); obj.matrix.copy(state.matrix); obj.matrix.decompose(obj.position, obj.quaternion, obj.scale); } delete obj.userData["image-tracking"]; } } } } } private readonly imageToObjectMap = new Map(); private readonly currentImages: WebXRTrackedImage[] = []; private readonly webXRIncubationsWarning = "Image tracking is currently not supported on this device. On Chrome for Android, you can enable the console.log('I')\">chrome://flags/#webxr-incubations flag."; /** @internal */ onUpdateXR(args: NeedleXREventArgs): void { this.currentImages.length = 0; const frame = args.xr.frame; if (!frame) return; if (!("getImageTrackingResults" in frame)) { if (!this["didPrintWarning"]) { this["didPrintWarning"] = true; console.log(this.webXRIncubationsWarning); } this._supported = false; showBalloonWarning(this.webXRIncubationsWarning); return; } // Check if enabled features (if available) contains image tracking - if it's not available this statement should not catch // This handles mobile VR with image tracking. Seems like the "getImageTrackingResults" is available on the frame object but then we get runtime exceptions because the feature is (in VR) not enabled else if (args.xr.session.enabledFeatures?.includes("image-tracking") === false) { // Image tracking is not enabled for this session return; } else if (frame.session && typeof frame.getImageTrackingResults === "function") { const results = frame.getImageTrackingResults(); if (results.length > 0) { const space = this.context.renderer.xr.getReferenceSpace(); if (space) { for (const result of results) { const state = result.trackingState; const imageIndex = result.index; const trackedImage = this.trackedImageIndexMap.get(imageIndex); if (trackedImage) { const pose = frame.getPose(result.imageSpace, space); const imageData = new WebXRTrackedImage(this, trackedImage, result.image, result.measuredSize, state, pose); this.currentImages.push(imageData); } else { if (debug) { console.warn("No tracked image for index", imageIndex); } } } if (this.currentImages.length > 0) { try { const eventData: WebXRImageTrackingEvent = { trackedImages: this.currentImages }; this.dispatchEvent(new CustomEvent("image-tracking", { detail: eventData })); this.imageTracked.invoke(eventData); this.onImageTrackingUpdate(this.currentImages); } catch (e) { console.error(e); } } } } } // disable any objects that are no longer tracked /** time in millis */ const hysteresis = 1000; for (const [key, value] of this.imageToObjectMap) { if (!value.object || !key) continue; // If the user disallowed hiding the object when tracking is lost, skip this if (key.hideWhenTrackingIsLost === false) continue; let found = false; for (const trackedImage of this.currentImages) { if (trackedImage.model === key) { // Make sure to keep the object visible if it's marked as static OR is tracked OR was tracked very recently (e.g. low framerate or bad tracking on device) const timeSinceLastTracking = Date.now() - value.lastTrackingTime; if(debug) showBalloonMessage(key.image + ", State: " + trackedImage.state + (key.imageDoesNotMove ? " (static)" : "") + (timeSinceLastTracking <= hysteresis ? " (hysteresis)" : "")); if (key.imageDoesNotMove || trackedImage.state === "tracked" || timeSinceLastTracking <= hysteresis) { found = true; break; } } } if (!found) { GameObject.setActive(value.object, false); } } } private onImageTrackingUpdate = (images: WebXRTrackedImage[]) => { const xr = NeedleXRSession.active; if (!xr) return; for (const image of images) { const model = image.model; const isTracked = image.state === "tracked"; // don't do anything if we don't have an object to track - can be handled externally through events if (!model.object) continue; let trackedData = this.imageToObjectMap.get(model); if (trackedData === undefined) { trackedData = { object: null, frames: 0, lastTrackingTime: Date.now() }; this.imageToObjectMap.set(model, trackedData); model.object.loadAssetAsync().then((asset: Object3D | null) => { if (model.createObjectInstance && asset) { asset = GameObject.instantiate(asset); } if (asset) { trackedData!.object = asset; // workaround for instancing currently not properly updating // instanced objects become visible when the image is recognized for the second time // we need to look into this further https://linear.app/needle/issue/NE-3936 for (const rend of asset.getComponentsInChildren(Renderer)) { rend.setInstancingEnabled(false); } // make sure to parent to the WebXR.rig if (xr.rig) { xr.rig.gameObject.add(asset); image.applyToObject(asset); if (!(asset as IGameObject).activeSelf) GameObject.setActive(asset, true); // InstancingUtil.markDirty(asset); } else { console.warn("XRImageTracking: missing XRRig"); } } }); } else { trackedData.frames++; if (isTracked) trackedData.lastTrackingTime = Date.now(); // TODO we could do a bit more here: e.g. sample for the first 1s or so of getting pose data // to improve the tracking quality a bit. if (model.imageDoesNotMove && trackedData.frames > 10) continue; if (!trackedData.object) continue; if (xr.rig) { xr.rig.gameObject.add(trackedData.object); image.applyToObject(trackedData.object, this.smooth ? this.context.time.deltaTimeUnscaled * 3 : undefined); if (!(trackedData.object as IGameObject).activeSelf) { GameObject.setActive(trackedData.object, true); } // InstancingUtil.markDirty(trackedData.object); } } } } }