import { Mesh, Object3D, Quaternion, Vector3 } from "three"; import { AssetReference } from "../../engine/engine_addressables.js"; import { ObjectUtils, PrimitiveType } from "../../engine/engine_create_objects.js"; import { ViewDevice } from "../../engine/engine_playerview.js"; import { serializable } from "../../engine/engine_serialization_decorator.js"; import type { IGameObject } from "../../engine/engine_types.js"; import { getParam, PromiseAllWithErrors } from "../../engine/engine_utils.js"; import { setCustomVisibility } from "../../engine/js-extensions/Layers.js"; import { NeedleXRController, type NeedleXREventArgs, NeedleXRSession, NeedleXRUtils } from "../../engine/xr/api.js"; import { PlayerState } from "../../engine-components-experimental/networking/PlayerSync.js"; import { Behaviour, GameObject } from "../Component.js"; import { SyncedTransform } from "../SyncedTransform.js"; import { AvatarMarker } from "./WebXRAvatar.js"; import { XRFlag } from "./XRFlag.js"; const debug = getParam("debugwebxr"); const flipForwardQuaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI); /** * Avatar component to setup a WebXR avatar with head and hand objects. * * The avatar will automatically synchronize the head and hand objects with the XR rig when entering an XR session. * * @summary WebXR Avatar component for head and hands synchronization * @category XR * @category Networking * @group Components */ export class Avatar extends Behaviour { @serializable(AssetReference) head?: AssetReference; @serializable(AssetReference) leftHand?: AssetReference; @serializable(AssetReference) rightHand?: AssetReference; private _leftHandMeshes?: Mesh[]; private _rightHandMeshes?: Mesh[]; // meshes of remote avatar hands - hidden render-only so the hand root stays active and keeps syncing private _remoteLeftHandMeshes?: Mesh[]; private _remoteRightHandMeshes?: Mesh[]; private _syncTransforms?: SyncedTransform[]; async onEnterXR(_args: NeedleXREventArgs) { if (!this.activeAndEnabled) return; if (debug) console.warn("AVATAR ENTER XR", this.guid, this.sourceId, this, this.activeAndEnabled) if (this._syncTransforms) this._syncTransforms.length = 0; await this.prepareAvatar(); const playerstate = PlayerState.getFor(this); if (playerstate?.owner) { const marker = this.gameObject.addComponent(AvatarMarker)!; marker.avatar = this.gameObject; marker.connectionId = playerstate.owner; this.context.players.setPlayerView(playerstate.owner, this.head?.asset, ViewDevice.Headset); } else if (this.context.connection.isConnected) console.error("No player state found for avatar", this); // don't destroy the avatar when entering XR and not connected to a networking backend else if (playerstate && !this.context.connection.isConnected) playerstate.dontDestroy = true; } onLeaveXR(_args: NeedleXREventArgs): void { const marker = this.gameObject.getComponent(AvatarMarker); if (marker) { marker.destroy(); } } onUpdateXR(args: NeedleXREventArgs): void { if (!this.activeAndEnabled) return; const isLocalPlayer = PlayerState.isLocalPlayer(this); if (!isLocalPlayer) return; const xr = args.xr; // make sure the avatar is inside the active rig if (xr.rig && xr.rig.gameObject !== this.gameObject.parent) { this.gameObject.position.set(0, 0, 0); this.gameObject.rotation.set(0, 0, 0); this.gameObject.scale.set(1, 1, 1); xr.rig.gameObject.add(this.gameObject); } // this.gameObject.position.copy(xr.rig!.gameObject.position); // this.gameObject.quaternion.copy(xr.rig!.gameObject.quaternion); // this.gameObject.scale.set(1, 1, 1); if (this._syncTransforms && isLocalPlayer) { for (const sync of this._syncTransforms) { sync.fastMode = true; if (!sync.isOwned()) sync.requestOwnership(); } } // synchronize head if (this.head && this.context.mainCamera) { const headObj = this.head.asset as IGameObject; headObj.position.copy(this.context.mainCamera.position); headObj.position.x *= -1; headObj.position.z *= -1; headObj.quaternion.copy(this.context.mainCamera.quaternion); headObj.quaternion.x *= -1; // HACK: XRFlag limitation workaround to make sure first person user head is never rendered if (this.context.time.frameCount % 10 === 0 && this.head.asset) { const xrflags = GameObject.getComponentsInChildren(this.head.asset, XRFlag); for (const flag of xrflags) { flag.enabled = false; flag.gameObject.visible = false; } } } // synchronize hands // IMPORTANT: never toggle `.visible` on the hand object itself - it carries the SyncedTransform, // and setting it invisible deactivates the component (onDisable -> freeOwnership), which means the // owner loses ownership and stops sending (the remote hand then stays frozen). The head syncs fine // precisely because its synced root is never deactivated (only a child mesh is hidden). So we keep // the hand object active and hide only the rendered meshes (render-only, via setCustomVisibility). const leftCtrl = args.xr.leftController; const leftObj = this.leftHand?.asset as Object3D; if (leftObj) { if (leftCtrl) { leftObj.position.copy(leftCtrl.gripPosition); leftObj.quaternion.copy(leftCtrl.gripQuaternion); leftObj.quaternion.multiply(flipForwardQuaternion); } leftObj.visible = true; this.updateHandVisibility(leftCtrl, leftObj, this._leftHandMeshes); } const right = args.xr.rightController; const rightObj = this.rightHand?.asset as Object3D; if (rightObj) { if (right) { rightObj.position.copy(right.gripPosition); rightObj.quaternion.copy(right.gripQuaternion); rightObj.quaternion.multiply(flipForwardQuaternion); } rightObj.visible = true; this.updateHandVisibility(right, rightObj, this._rightHandMeshes); } } onBeforeRender(): void { // Remote avatar visibility depends on the *owner's* tracking state, not on whether THIS client is // in XR: other users' avatars are syncInstantiated onto non-XR clients too (spectators / lobby), // and must hide/show their hands correctly there as well. So this runs regardless of the local XR // state. updateRemoteAvatarVisibility() early-outs for the local player and when not connected. if (this.context.time.frame % 10 === 0) this.updateRemoteAvatarVisibility(); } private updateHandVisibility(controller: NeedleXRController | null | undefined, avatarHand: Object3D, meshes: Mesh[] | undefined) { if (meshes) { // Hide the hand meshes (render-only) when the hand is not tracked, or when another model // (e.g. the controller/hand model) is being rendered in its place. We hide the meshes rather // than the object's `visible` flag because that would disable SyncedTransform networking. const isTracking = controller?.isTracking === true; const hasOtherRenderingModel = !!controller?.model && controller.model.visible && controller.model !== avatarHand; const render = isTracking && !hasOtherRenderingModel; meshes.forEach(mesh => { setCustomVisibility(mesh, render); }); } } private updateRemoteAvatarVisibility() { if (this.context.connection.isConnected) { const state = PlayerState.getFor(this); if (state && state.isLocalPlayer == false) { const sync = NeedleXRSession.getXRSync(this.context); if (sync) { if (sync.hasState(state.owner)) { this.tryFindAvatarObjectsIfMissing(); // Keep the hand object active (don't toggle `.visible`) so its SyncedTransform keeps // applying updates and a non-owner freeOwnership can't strip the owner's ownership. // Hide/show the rendered meshes based on the owner's tracking state instead. const leftRendered = sync.isTracking(state.owner, "left") ?? false; this._remoteLeftHandMeshes = this.setRemoteHandRendered(this.leftHand?.asset as Object3D, this._remoteLeftHandMeshes, leftRendered); const rightRendered = sync.isTracking(state.owner, "right") ?? false; this._remoteRightHandMeshes = this.setRemoteHandRendered(this.rightHand?.asset as Object3D, this._remoteRightHandMeshes, rightRendered); } } // HACK: XRFlag limitation workaround to make sure first person user head of OTHER users is ALWAYS rendered if (this.head?.asset) { const xrflags = GameObject.getComponentsInChildren(this.head.asset, XRFlag); for (const flag of xrflags) { flag.enabled = false; flag.gameObject.visible = true; } } } } } /** * Show/hide a remote avatar hand render-only. The hand object stays active so its SyncedTransform keeps * applying networked updates (toggling `.visible` would deactivate it and could free the owner's ownership). * The mesh list is gathered lazily and cached, and returned so the caller can keep it. */ private setRemoteHandRendered(obj: Object3D | undefined, meshes: Mesh[] | undefined, rendered: boolean): Mesh[] | undefined { if (!obj) return meshes; obj.visible = true; if (!meshes || meshes.length === 0) { meshes = []; obj.traverse(o => { if ((o as Mesh).isMesh) meshes!.push(o as Mesh); }); } for (const mesh of meshes) setCustomVisibility(mesh, rendered); return meshes; } private tryFindAvatarObjectsIfMissing() { // if no avatar objects are set, try to find them if (!this.head || !this.leftHand || !this.rightHand) { const res = { head: this.head, leftHand: this.leftHand, rightHand: this.rightHand }; NeedleXRUtils.tryFindAvatarObjects(this.gameObject, this.sourceId || "", res); if (res.head) this.head = res.head; if (res.leftHand) this.leftHand = res.leftHand; if (res.rightHand) this.rightHand = res.rightHand; } } private async prepareAvatar() { // if no avatar objects are set, try to find them this.tryFindAvatarObjectsIfMissing(); if (!this.head) { const head = new Object3D(); head.name = "Head"; const cube = ObjectUtils.createPrimitive(PrimitiveType.Cube); head.add(cube); this.gameObject.add(head); this.head = new AssetReference("", this.sourceId, head); if (debug) console.log("Create head", head); } else if (this.head instanceof Object3D) { this.head = new AssetReference("", this.sourceId, this.head); } if (!this.rightHand) { const rightHand = new Object3D(); rightHand.name = "Right Hand"; this.gameObject.add(rightHand); this.rightHand = new AssetReference("", this.sourceId, rightHand); if (debug) console.log("Create right hand", rightHand); } else if (this.rightHand instanceof Object3D) { this.rightHand = new AssetReference("", this.sourceId, this.rightHand); } if (!this.leftHand) { const leftHand = new Object3D(); leftHand.name = "Left Hand"; this.gameObject.add(leftHand); this.leftHand = new AssetReference("", this.sourceId, leftHand); if (debug) console.log("Create left hand", leftHand); } else if (this.leftHand instanceof Object3D) { this.leftHand = new AssetReference("", this.sourceId, this.leftHand); } await this.loadAvatarObjects(this.head, this.leftHand, this.rightHand); this._leftHandMeshes = []; this.leftHand.asset?.traverse((obj) => { if ((obj as Mesh)?.isMesh) this._leftHandMeshes!.push(obj as Mesh); }); this._rightHandMeshes = []; this.rightHand.asset?.traverse((obj) => { if ((obj as Mesh)?.isMesh) this._rightHandMeshes!.push(obj as Mesh); }); if (PlayerState.isLocalPlayer(this.gameObject)) { this._syncTransforms = GameObject.getComponentsInChildren(this.gameObject, SyncedTransform); } } private async loadAvatarObjects(head: AssetReference, left: AssetReference, right: AssetReference) { const pHead = head.loadAssetAsync(); const pHandLeft = left.loadAssetAsync(); const pHandRight = right.loadAssetAsync(); const promises = new Array>(); if (pHead) promises.push(pHead); if (pHandLeft) promises.push(pHandLeft); if (pHandRight) promises.push(pHandRight); const res = await PromiseAllWithErrors(promises); if (debug) console.log("Avatar loaded results:", res); } }