import { Builder } from "flatbuffers"; import { Camera as ThreeCamera, Object3D, Quaternion, Vector3 } from "three"; import { isDevEnvironment } from "../engine/debug/index.js"; import { AssetReference } from "../engine/engine_addressables.js"; import { InstantiateOptions } from "../engine/engine_gameobject.js"; import { InstancingUtil } from "../engine/engine_instancing.js"; import { NetworkConnection } from "../engine/engine_networking.js"; import { ViewDevice } from "../engine/engine_playerview.js"; import { serializable } from "../engine/engine_serialization_decorator.js"; import * as utils from "../engine/engine_three_utils.js" import { registerBinaryType } from "../engine-schemes/schemes.js"; import { SyncedCameraModel } from "../engine-schemes/synced-camera-model.js"; import { Vec3 } from "../engine-schemes/vec3.js"; import { Behaviour, GameObject } from "./Component.js"; import { AvatarMarker } from "./webxr/WebXRAvatar.js"; const SyncedCameraModelIdentifier = "SCAM"; registerBinaryType(SyncedCameraModelIdentifier, SyncedCameraModel.getRootAsSyncedCameraModel); const builder = new Builder(); // enum CameraSyncEvent { // Update = "sync-update-camera", // } class CameraModel { userId: string; guid: string; // dontSave: boolean = true; // pos: { x: number, y: number, z: number } = { x: 0, y: 0, z: 0 }; // rot: { x: number, y: number, z: number } = { x: 0, y: 0, z: 0 }; constructor(connectionId: string, guid: string) { this.guid = guid; this.userId = connectionId; } send(cam: ThreeCamera | null | undefined, con: NetworkConnection) { if (cam) { builder.clear(); const guid = builder.createString(this.guid); const userId = builder.createString(this.userId); SyncedCameraModel.startSyncedCameraModel(builder); SyncedCameraModel.addGuid(builder, guid); SyncedCameraModel.addUserId(builder, userId); const p = utils.getWorldPosition(cam); const r = utils.getWorldRotation(cam); SyncedCameraModel.addPos(builder, Vec3.createVec3(builder, p.x, p.y, p.z)); SyncedCameraModel.addRot(builder, Vec3.createVec3(builder, r.x, r.y, r.z)); const offset = SyncedCameraModel.endSyncedCameraModel(builder); builder.finish(offset, SyncedCameraModelIdentifier); con.sendBinary(builder.asUint8Array()); } } } declare type UserCamInfo = { obj: Object3D, lastUpdate: number; userId: string; }; /** * SyncedCamera is a component that syncs the camera position and rotation of all users in the room. * A prefab can be set to represent the remote cameras visually in the scene. * @category Networking * @group Components */ export class SyncedCamera extends Behaviour { static instances: UserCamInfo[] = []; getCameraObject(userId: string): Object3D | null { const guid = this.userToCamMap[userId]; if (!guid) return null; return this.remoteCams[guid].obj; } /** * The prefab to visually represent the remote cameras in the scene. */ @serializable([Object3D, AssetReference]) public cameraPrefab: Object3D | null | AssetReference = null; private _lastWorldPosition!: Vector3; private _lastWorldQuaternion!: Quaternion; private _model: CameraModel | null = null; private _needsUpdate: boolean = true; private _lastUpdateTime: number = 0; private remoteCams: { [id: string]: UserCamInfo } = {}; private userToCamMap: { [id: string]: string } = {}; private _camTimeoutInSeconds = 10; private _receiveCallback: Function | null = null; /** @internal */ async awake() { this._lastWorldPosition = this.worldPosition.clone(); this._lastWorldQuaternion = this.worldQuaternion.clone(); if (this.cameraPrefab) { if ("uri" in this.cameraPrefab) { this.cameraPrefab = await this.cameraPrefab.instantiate(this.gameObject); } if (this.cameraPrefab && "isObject3D" in this.cameraPrefab) { this.cameraPrefab.visible = false; } } } /** @internal */ onEnable(): void { this._receiveCallback = this.context.connection.beginListenBinary(SyncedCameraModelIdentifier, this.onReceivedRemoteCameraInfoBin.bind(this)); } /** @internal */ onDisable(): void { this.context.connection.stopListenBinary(SyncedCameraModelIdentifier, this._receiveCallback); } /** @internal */ update(): void { for (const guid in this.remoteCams) { const cam = this.remoteCams[guid]; const timeDiff = this.context.time.realtimeSinceStartup - cam.lastUpdate; if (!cam || (timeDiff) > this._camTimeoutInSeconds) { if (isDevEnvironment()) console.log("Remote cam timeout", guid); if (cam?.obj) { GameObject.destroy(cam.obj); } delete this.remoteCams[guid]; if (cam) delete this.userToCamMap[cam.userId]; SyncedCamera.instances.push(cam); this.context.players.removePlayerView(cam.userId, ViewDevice.Browser); continue; } } if (this.context.isInXR) return; const cam = this.context.mainCamera if (cam === null) { this.enabled = false; return; } if (!this.context.connection.isConnected || this.context.connection.connectionId === null) return; if (this._model === null) { this._model = new CameraModel(this.context.connection.connectionId, this.context.connection.connectionId + "_camera"); } const wp = utils.getWorldPosition(cam); const wq = utils.getWorldQuaternion(cam); if (wp.distanceTo(this._lastWorldPosition) > 0.001 || wq.angleTo(this._lastWorldQuaternion) > 0.01) { this._needsUpdate = true; } this._lastWorldPosition.copy(wp); this._lastWorldQuaternion.copy(wq); if (!this._needsUpdate || this.context.time.frameCount % 2 !== 0) { if (this.context.time.realtimeSinceStartup - this._lastUpdateTime > this._camTimeoutInSeconds * .5) { // send update anyways to avoid timeout } else return; } this._lastUpdateTime = this.context.time.realtimeSinceStartup; this._needsUpdate = false; this._model.send(cam, this.context.connection); if (!this.context.isInXR) this.context.players.setPlayerView(this.context.connection.connectionId, cam, ViewDevice.Browser); } private onReceivedRemoteCameraInfoBin(model: SyncedCameraModel) { const guid = model.guid(); if (!guid) return; const userId = model.userId(); if (!userId) return; if (!this.context.connection.userIsInRoom(userId)) return; if (!this.cameraPrefab) return; let rc = this.remoteCams[guid]; if (!rc) { if ("isObject3D" in this.cameraPrefab) { const opt = new InstantiateOptions(); opt.context = this.context; const instance = GameObject.instantiate(this.cameraPrefab, opt) as GameObject; rc = this.remoteCams[guid] = { obj: instance, lastUpdate: this.context.time.realtimeSinceStartup, userId: userId }; rc.obj.visible = true; this.gameObject.add(instance); this.userToCamMap[userId] = guid; SyncedCamera.instances.push(rc); const marker = GameObject.getOrAddComponent(instance, AvatarMarker); marker.connectionId = userId; marker.avatar = instance; } else { return; } // console.log(this.remoteCams); } const obj = rc.obj; this.context.players.setPlayerView(userId, obj, ViewDevice.Browser); rc.lastUpdate = this.context.time.realtimeSinceStartup; InstancingUtil.markDirty(obj); const pos = model.pos(); if (pos) utils.setWorldPositionXYZ(obj, pos.x(), pos.y(), pos.z()); const rot = model.rot(); if (rot) utils.setWorldRotationXYZ(obj, rot.x(), rot.y(), rot.z()); } }