import { isDevEnvironment, showBalloonError, showBalloonWarning } from "../engine/debug/index.js"; import { Application } from "../engine/engine_application.js"; import { RoomEvents } from "../engine/engine_networking.js"; import { disposeStream, NetworkedStreamEvents, NetworkedStreams, StreamEndedEvent, StreamReceivedEvent } from "../engine/engine_networking_streams.js" import { serializable } from "../engine/engine_serialization_decorator.js"; import { delay, DeviceUtilities, getParam } from "../engine/engine_utils.js"; import { getIconElement } from "../engine/webcomponents/icons.js"; import { Behaviour } from "./Component.js"; import { EventList } from "./EventList.js"; export const noVoip = "noVoip"; const debugParam = getParam("debugvoip"); /** * [Voip](https://engine.needle.tools/docs/api/Voip) Voice over IP (VoIP) component for real-time audio communication between users. * Allows sending and receiving audio streams in networked rooms. * * **Requirements:** * - Active network connection (via {@link SyncedRoom} or manual connection) * - User permission for microphone access (requested automatically) * - HTTPS connection (required for WebRTC) * * **Features:** * - Automatic connection when joining rooms (`autoConnect`) * - Background audio support (`runInBackground`) * - Optional UI toggle button (`createMenuButton`) * - Mute/unmute control * * **Debug:** Use `?debugvoip` URL parameter or set `debug = true` for logging. * Press 'v' to toggle mute, 'c' to connect/disconnect when debugging. * * @example Enable VoIP in your scene * ```ts * const voip = myObject.addComponent(Voip); * voip.autoConnect = true; * voip.createMenuButton = true; * * // Manual control * voip.connect(); // Start sending your microphone * voip.disconnect(); // Stop sending your microphone * voip.setMuted(true); // Mute incoming audio (silence other users) * ``` * * @summary Voice over IP for networked audio communication * @category Networking * @group Components * @see {@link SyncedRoom} for room management * @see {@link NetworkedStreams} for the underlying streaming * @see {@link ScreenCapture} for video streaming * @link https://engine.needle.tools/docs/networking.html */ export class Voip extends Behaviour { /** When enabled, VoIP will start when a room is joined or when this component is enabled while already in a room. * @default true */ @serializable() autoConnect: boolean = true; /** * When enabled, VoIP will stay connected even when the browser tab is not focused/active anymore. * @default true */ @serializable() runInBackground: boolean = true; /** * When enabled, a menu button will be created to allow the user to toggle VoIP on and off. */ @serializable() createMenuButton: boolean = true; /** * When enabled debug messages will be printed to the console. This is useful for debugging audio issues. You can also append ?debugvoip to the URL to enable this. */ debug: boolean = false; private _volume: number = 1; /** * Volume for incoming audio streams (0 = silent, 1 = full volume). * Changes apply immediately to all active incoming streams. */ @serializable() get volume(): number { return this._volume; } set volume(val: number) { // HTMLMediaElement.volume throws IndexSizeError outside [0,1] — clamp before assigning. // Reject NaN so we don't poison _volume and break serialization. if (Number.isNaN(val)) return; const clamped = Math.max(0, Math.min(1, val)); this._volume = clamped; for (const audio of this._incomingStreams.values()) { audio.volume = clamped; } } /** * Get the incoming audio element for a specific user. * Use this to route audio through the Web Audio API for spatial audio, effects, or analysis. * @param userId The connection ID of the remote user * @returns The HTMLAudioElement for this user's stream, or undefined if not connected * @example * ```ts * const audioEl = voip.getAudioElement(userId); * if (audioEl) { * const audioCtx = new AudioContext(); * const source = audioCtx.createMediaElementSource(audioEl); * const panner = audioCtx.createPanner(); * source.connect(panner).connect(audioCtx.destination); * } * ``` */ getAudioElement(userId: string): HTMLAudioElement | undefined { return this._incomingStreams.get(userId); } /** * Get all active incoming audio streams as a Map of userId → HTMLAudioElement. * Useful for iterating over all connected voice users. */ get incomingStreams(): ReadonlyMap { return this._incomingStreams; } /** * Normalized amplitude threshold for speaking detection (0–1). When a user's average * audio amplitude exceeds this, they are considered "speaking". Default is 0.1. */ @serializable() speakingThreshold: number = 0.1; /** * Event fired when a user's speaking state changes. * Passes `{ userId: string, isSpeaking: boolean, volume: number }`. */ @serializable(EventList) onSpeakingChanged: EventList = new EventList(); private _speakingStates = new Map(); private _analysers = new Map(); // Single shared AudioContext for all remote-user analysers. Browsers cap live AudioContexts at ~6 per tab, // so one-per-user would break voice rooms of 7+ participants. Lazily created on first setupAnalyser. private _sharedAudioContext?: AudioContext; private _lastSpeakingPollMs = 0; private _net?: NetworkedStreams; private _menubutton?: HTMLElement; /** @internal */ awake() { if (debugParam) this.debug = true; if (this.debug) { console.log("VOIP debugging: press 'v' to toggle incoming mute, 'c' to toggle connect/disconnect"); window.addEventListener("keydown", async (evt) => { const key = evt.key.toLowerCase(); switch (key) { case "v": console.log("VOIP: toggle incoming mute → ", !this.isMuted); this.setMuted(!this.isMuted); break; case "c": if (this.isSending) this.disconnect(); else this.connect(); break; } }); } } /** @internal */ onEnable(): void { if (!this._net) this._net = NetworkedStreams.create(this); if (this.debug) this._net.debug = true; this._net.addEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream); this._net.addEventListener(NetworkedStreamEvents.StreamEnded, this.onStreamEnded); this._net.enable(); if (this.autoConnect) { if (this.context.connection.isConnected) this.connect(); } this.context.connection.beginListen(RoomEvents.JoinedRoom, this.onJoinedRoom); this.context.connection.beginListen(RoomEvents.LeftRoom, this.onLeftRoom); this.onEnabledChanged(); this.updateButton(); window.addEventListener("visibilitychange", this.onVisibilityChanged); } /** @internal */ onDisable(): void { if (this._net) { this._net.stopSendingStream(this._outputStream); //@ts-ignore this._net.removeEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream); //@ts-ignore this._net.removeEventListener(NetworkedStreamEvents.StreamEnded, this.onStreamEnded) this._net?.disable(); } this.context.connection.stopListen(RoomEvents.JoinedRoom, this.onJoinedRoom); this.context.connection.stopListen(RoomEvents.LeftRoom, this.onLeftRoom); this.onEnabledChanged(); this.updateButton(); window.removeEventListener("visibilitychange", this.onVisibilityChanged); // Clean up analysers and the shared AudioContext (lazily recreated on next setupAnalyser). for (const userId of [...this._analysers.keys()]) { this.cleanupAnalyser(userId); } this.closeSharedAudioContext(); } /** @internal */ onDestroy(): void { this._menubutton?.remove(); this._menubutton = undefined; // Clean up all streams and analysers for (const userId of [...this._analysers.keys()]) { this.cleanupAnalyser(userId); } this.closeSharedAudioContext(); for (const incoming of this._incomingStreams.values()) { disposeStream(incoming.srcObject as MediaStream); } this._incomingStreams.clear(); this._speakingStates.clear(); } /** Set via the mic button (e.g. when the websocket connection closes and rejoins but the user was muted before we don't want to enable VOIP again automatically) */ private _allowSending = true; private _outputStream: MediaStream | null = null; // Tracks an in-flight connect() so concurrent callers coalesce onto the same promise // instead of each running getUserMedia in parallel (which would leak a MediaStream and // briefly transmit then kill the first acquired stream). private _connectInFlight?: Promise; /** * @returns true if the component is currently sending audio */ get isSending() { return this._outputStream != null && this._outputStream.active; } /** Start sending audio. */ connect(audioSource?: MediaTrackConstraints): Promise { // Coalesce concurrent callers. Without this, two near-simultaneous connect() calls // each call getUserMedia in parallel, the first stream gets disposed mid-broadcast // by the second, and a MediaStream leaks. if (this._connectInFlight) return this._connectInFlight; this._connectInFlight = this._connectImpl(audioSource).finally(() => { this._connectInFlight = undefined; }); return this._connectInFlight; } private async _connectImpl(audioSource?: MediaTrackConstraints): Promise { if (!this._net) { console.error("Cannot connect to voice chat - NetworkedStreams not initialized. Make sure the component is enabled before calling this method."); return false; } if (!this.context.connection.isConnected) { console.error("Cannot connect to voice chat - not connected to server"); this.updateButton(); return false; } else if (!await DeviceUtilities.microphonePermissionsGranted()) { console.error("Cannot connect to voice chat - microphone permissions not granted"); this.updateButton(); return false; } this._allowSending = true; this._net?.stopSendingStream(this._outputStream); disposeStream(this._outputStream); this._outputStream = await this.getAudioStream(audioSource); if (this._outputStream) { if (this.debug) console.log("VOIP: Got audio stream"); this._net?.startSendingStream(this._outputStream); this.updateButton(); return true; } else { this.updateButton(); if (!await DeviceUtilities.microphonePermissionsGranted()) { showBalloonError("Microphone permissions not granted: Please grant microphone permissions to use voice chat"); } else console.error("VOIP: Could not get audio stream - please make sure to connect an audio device and grant microphone permissions"); } if (this.debug || isDevEnvironment()) console.log("VOIP: Failed to get audio stream"); return false; } /** Stop sending audio (muting your own microphone) */ disconnect(opts?: { remember: boolean }) { if (opts?.remember) { this._allowSending = false; } this._net?.stopSendingStream(this._outputStream); disposeStream(this._outputStream); this._outputStream = null; this.updateButton(); } /** * Mute or unmute the audio you hear from other users (incoming streams). * This does NOT mute your own microphone — use {@link disconnect} to stop sending your microphone. */ setMuted(mute: boolean) { for (const audio of this._incomingStreams.values()) { audio.muted = mute; } } /** * Returns true if incoming audio is currently muted (you can't hear other users). * When there are no incoming streams, returns false. */ get isMuted() { if (this._incomingStreams.size === 0) return false; for (const audio of this._incomingStreams.values()) { if (!audio.muted) return false; } return true; } private async updateButton() { if (this.createMenuButton) { if (!this._menubutton) { this._menubutton = document.createElement("button"); this._menubutton.addEventListener("click", () => { if (this.isSending) { this.disconnect({ remember: true }); } else this.connect(); DeviceUtilities.microphonePermissionsGranted().then(res => { if (!res) showBalloonWarning("Microphone permissions not granted. Please allow your browser to use the microphone to be able to talk. Click on the button on the left side of your browser's address bar to allow microphone permissions."); }) }); } if (this._menubutton) { this.context.menu.appendChild(this._menubutton); if (this.activeAndEnabled) { this._menubutton.style.display = ""; } else { this._menubutton.style.display = "none"; } this._menubutton.title = this.isSending ? "Click to disable your microphone" : "Click to enable your microphone"; let label = this.isSending ? "" : ""; let icon = this.isSending ? "mic" : "mic_off"; const hasPermission = await DeviceUtilities.microphonePermissionsGranted(); if (!hasPermission) { label = "No Permission"; icon = "mic_off"; this._menubutton.title = "Microphone permissions not granted. Please allow your browser to use the microphone to be able to talk. This can usually be done in the addressbar of the webpage."; } this._menubutton.innerText = label; this._menubutton.prepend(getIconElement(icon)); if (this.context.connection.isConnected == false) this._menubutton.setAttribute("disabled", ""); else this._menubutton.removeAttribute("disabled"); } } else if (!this.activeAndEnabled) { this._menubutton?.remove(); } } // private _analyzer?: AudioAnalyser; /** @deprecated */ public getFrequency(_userId: string | null): number | null { if (!this["unsupported_getfrequency"]) { this["unsupported_getfrequency"] = true; if (isDevEnvironment()) showBalloonWarning("VOIP: getFrequency is currently not supported"); console.warn("VOIP: getFrequency is currently not supported"); } // null is get the first with some data // if (userId === null) { // for (const c in this._incomingStreams) { // const call = this._incomingStreams[c]; // if (call && call.currentAnalyzer) return call.currentAnalyzer.getAverageFrequency(); // } // return null; // } // const call = this._incomingStreams.get(userId); // if (call && call.currentAnalyzer) return call.currentAnalyzer.getAverageFrequency(); return null; } private async getAudioStream(audio?: MediaTrackConstraints) { if (!navigator.mediaDevices.getUserMedia) { console.error("No getDisplayMedia support"); return null; } const getUserMedia = async (constraints?: MediaTrackConstraints): Promise => { return await navigator.mediaDevices.getUserMedia({ audio: constraints ?? true, video: false }) .catch((err) => { console.warn("VOIP failed getting audio stream", err); return null; }); } const stream = await getUserMedia(audio); if (!stream) return null; // NE-5445, on iOS after calling `getUserMedia` it automatically switches the audio to the built-in microphone and speakers even if headphones are connected // if there's no device selected explictly we will try to automatically select an external device if (DeviceUtilities.isiOS() && audio?.deviceId === undefined) { const devices = await navigator.mediaDevices.enumerateDevices(); // select anything that doesn't have "iPhone" is likely "AirPods" or other bluetooth headphones const nonBuiltInAudioSource = devices.find((device) => (device.kind === "audioinput" || device.kind === "audiooutput") && !device.label.includes("iPhone")); if (nonBuiltInAudioSource) { const constraints = Object.assign({}, audio); constraints.deviceId = nonBuiltInAudioSource.deviceId; const externalStream = await getUserMedia(constraints); if (externalStream) { // Release the built-in mic stream we grabbed first — otherwise its tracks stay live. disposeStream(stream); return externalStream; } // External device acquisition failed — keep the original stream rather than returning null. } } return stream; } // we have to wait for the user to connect to a room when "auto connect" is enabled private onJoinedRoom = async () => { if (this.debug) console.log("VOIP: Joined room"); // Wait a moment for user list to be populated await delay(300) if (this.autoConnect && !this.isSending && this._allowSending) { this.connect(); } } private onLeftRoom = () => { if (this.debug) console.log("VOIP: Left room"); this.disconnect(); for (const incoming of this._incomingStreams.values()) { disposeStream(incoming.srcObject as MediaStream); } this._incomingStreams.clear(); for (const userId of this._analysers.keys()) { this.cleanupAnalyser(userId); } } private _incomingStreams: Map = new Map(); /** @internal */ update() { // Only run speaking detection if someone is listening if (!this.onSpeakingChanged || this.onSpeakingChanged.listenerCount <= 0) return; // Rate-limit analysis to ~10Hz. Speaking-state is a coarse UI signal; running FFT per // remote user every frame (60Hz) is wasteful and scales linearly with participant count. const now = performance.now(); if (now - this._lastSpeakingPollMs < 100) return; this._lastSpeakingPollMs = now; for (const [userId, info] of this._analysers) { info.analyser.getByteFrequencyData(info.data as Uint8Array); // Average amplitude normalized to 0–1 let sum = 0; for (let i = 0; i < info.data.length; i++) sum += info.data[i]; const volume = sum / info.data.length / 255; const wasSpeaking = this._speakingStates.get(userId) ?? false; const isSpeaking = volume > this.speakingThreshold; if (isSpeaking !== wasSpeaking) { this._speakingStates.set(userId, isSpeaking); this.onSpeakingChanged.invoke({ userId, isSpeaking, volume }); } } } private setupAnalyser(userId: string, _audioElement: HTMLAudioElement, stream: MediaStream) { // Only set up if someone is listening or might listen if (this._analysers.has(userId)) return; try { if (!this._sharedAudioContext) { this._sharedAudioContext = new AudioContext(); } const ctx = this._sharedAudioContext; const source = ctx.createMediaStreamSource(stream); const analyser = ctx.createAnalyser(); analyser.fftSize = 256; source.connect(analyser); const data = new Uint8Array(analyser.frequencyBinCount); this._analysers.set(userId, { source, analyser, data }); } catch (err) { if (this.debug) console.warn("VOIP: Failed to create analyser for", userId, err); } } private cleanupAnalyser(userId: string) { const info = this._analysers.get(userId); if (info) { info.source.disconnect(); info.analyser.disconnect(); this._analysers.delete(userId); } this._speakingStates.delete(userId); } private closeSharedAudioContext() { if (this._sharedAudioContext) { this._sharedAudioContext.close(); this._sharedAudioContext = undefined; } } private onReceiveStream = (evt: StreamReceivedEvent) => { const userId = evt.target.userId; const stream = evt.stream; let audioElement = this._incomingStreams.get(userId); if (!audioElement) { audioElement = new Audio() this._incomingStreams.set(userId, audioElement); } audioElement.srcObject = stream; audioElement.volume = this._volume; audioElement.setAttribute("autoplay", "true"); this.setupAnalyser(userId, audioElement, stream); // for mobile we need to wait for user interaction to play audio. Auto play doesnt work on android when the page is refreshed Application.registerWaitForInteraction(() => { audioElement?.play().catch((err) => { console.error("VOIP: Failed to play audio", err); }); }) } private onStreamEnded = (evt: StreamEndedEvent) => { const existing = this._incomingStreams.get(evt.userId); disposeStream(existing?.srcObject as MediaStream); this._incomingStreams.delete(evt.userId); this.cleanupAnalyser(evt.userId); } private onEnabledChanged = () => { for (const key of this._incomingStreams) { const element = key[1]; element.muted = !this.enabled; } } private onVisibilityChanged = () => { if (this.runInBackground) return; const muted = document.visibilityState !== "visible"; // Mute incoming so we don't hear other users while tab is hidden. this.setMuted(muted); // Also disable our outgoing mic tracks (cheaper than disconnect/reconnect — keeps the mic permission). const tracks = this._outputStream?.getAudioTracks(); if (tracks) for (const t of tracks) t.enabled = !muted; }; }