import { AudioTrackFeature } from '@livekit/protocol'; import { TrackInvalidError } from '../errors'; import { TrackEvent } from '../events'; import { computeBitrate, monitorFrequency } from '../stats'; import type { AudioSenderStats } from '../stats'; import type { LoggerOptions } from '../types'; import { isReactNative, isWeb } from '../utils'; import LocalTrack from './LocalTrack'; import { Track } from './Track'; import type { AudioCaptureOptions } from './options'; import type { AudioProcessorOptions, TrackProcessor } from './processor/types'; import { constraintsForOptions, detectSilence } from './utils'; export default class LocalAudioTrack extends LocalTrack { /** @internal */ stopOnMute: boolean = false; private prevStats?: AudioSenderStats; private isKrispNoiseFilterEnabled = false; /** * the processed track we currently have krisp feature listeners registered on. Tracked * separately, as `processor.processedTrack` can be swapped out while we hold the listeners. */ private observedProcessedTrack?: MediaStreamTrack; protected processor?: TrackProcessor | undefined; /** * boolean indicating whether enhanced noise cancellation is currently being used on this track */ get enhancedNoiseCancellation() { return this.isKrispNoiseFilterEnabled; } /** * * @param mediaTrack * @param constraints MediaTrackConstraints that are being used when restarting or reacquiring tracks * @param userProvidedTrack Signals to the SDK whether or not the mediaTrack should be managed (i.e. released and reacquired) internally by the SDK */ constructor( mediaTrack: MediaStreamTrack, constraints?: MediaTrackConstraints, userProvidedTrack = true, audioContext?: AudioContext, loggerOptions?: LoggerOptions, ) { super(mediaTrack, Track.Kind.Audio, constraints, userProvidedTrack, loggerOptions); this.audioContext = audioContext; this.checkForSilence(); } async mute(): Promise { const unlock = await this.muteLock.lock(); try { if (this.isMuted) { this.log.debug('Track already muted', this.logContext); return this; } // disabled special handling as it will cause BT headsets to switch communication modes if (this.source === Track.Source.Microphone && this.stopOnMute && !this.isUserProvided) { this.log.debug('stopping mic track', this.logContext); // also stop the track, so that microphone indicator is turned off this._mediaStreamTrack.stop(); } await super.mute(); return this; } finally { unlock(); } } async unmute(): Promise { const unlock = await this.muteLock.lock(); try { if (!this.isMuted) { this.log.debug('Track already unmuted', this.logContext); return this; } if ( this.source === Track.Source.Microphone && (this.stopOnMute || this._mediaStreamTrack.readyState === 'ended' || this.pendingDeviceChange) && !this.isUserProvided ) { this.log.debug('reacquiring mic track', this.logContext); await this.restart(undefined, true); } await super.unmute(); return this; } finally { unlock(); } } async restartTrack(options?: AudioCaptureOptions) { let constraints: MediaTrackConstraints | undefined; if (options) { const streamConstraints = constraintsForOptions({ audio: options }); if (typeof streamConstraints.audio !== 'boolean') { constraints = streamConstraints.audio; } } await this.restart(constraints); } async applyConstraints( constraints: Pick< AudioCaptureOptions, 'autoGainControl' | 'noiseSuppression' | 'echoCancellation' | 'voiceIsolation' >, ): Promise { const unlock = await this.trackChangeLock.lock(); try { const res = await this._mediaStreamTrack.applyConstraints(constraints); this._constraints = { ...this._constraints, ...constraints }; return res; } finally { unlock(); } } protected async restart( constraints?: MediaTrackConstraints, isUnmuting?: boolean, ): Promise { const track = await super.restart(constraints, isUnmuting); this.checkForSilence(); return track; } /* @internal */ startMonitor() { if (!isWeb()) { return; } if (this.monitorInterval) { return; } this.monitorInterval = setInterval(() => { this.monitorSender(); }, monitorFrequency); } protected monitorSender = async () => { if (!this.sender) { this._currentBitrate = 0; return; } let stats: AudioSenderStats | undefined; try { stats = await this.getSenderStats(); } catch (e) { this.log.error('could not get audio sender stats', { ...this.logContext, error: e }); return; } if (stats && this.prevStats) { this._currentBitrate = computeBitrate(stats, this.prevStats); } this.prevStats = stats; }; private handleKrispNoiseFilterEnable = () => { this.isKrispNoiseFilterEnabled = true; this.log.debug(`Krisp noise filter enabled`, this.logContext); this.emit( TrackEvent.AudioTrackFeatureUpdate, this, AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION, true, ); }; private handleKrispNoiseFilterDisable = () => { this.isKrispNoiseFilterEnabled = false; this.log.debug(`Krisp noise filter disabled`, this.logContext); this.emit( TrackEvent.AudioTrackFeatureUpdate, this, AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION, false, ); }; /** * registers the krisp feature listeners on the given processed track, moving them over from * a previously observed one. Passing `undefined` just removes the existing listeners. */ private observeProcessedTrack(processedTrack: MediaStreamTrack | undefined) { if (this.observedProcessedTrack === processedTrack) { return; } if (this.observedProcessedTrack) { this.observedProcessedTrack.removeEventListener( 'enable-lk-krisp-noise-filter', this.handleKrispNoiseFilterEnable, ); this.observedProcessedTrack.removeEventListener( 'disable-lk-krisp-noise-filter', this.handleKrispNoiseFilterDisable, ); this.observedProcessedTrack = undefined; if (this.isKrispNoiseFilterEnabled) { // we're no longer observing the track that had the filter enabled, so the feature // can't be active anymore. Make sure the state doesn't get stuck on `true`. this.handleKrispNoiseFilterDisable(); } } if (!processedTrack) { return; } processedTrack.addEventListener( 'enable-lk-krisp-noise-filter', this.handleKrispNoiseFilterEnable, ); processedTrack.addEventListener( 'disable-lk-krisp-noise-filter', this.handleKrispNoiseFilterDisable, ); this.observedProcessedTrack = processedTrack; } /** * Sets a processor on this track. * * The processor is destroyed when the track is stopped, and also when the audio context goes * away, so a disconnect ends it on both counts and it has to be set again after reconnecting. * See {@link TrackProcessor} for the full lifecycle. */ async setProcessor(processor: TrackProcessor) { const unlock = await this.trackChangeLock.lock(); const stops = this.stopCount; try { if (!isReactNative() && !this.audioContext) { throw Error( 'Audio context needs to be set on LocalAudioTrack in order to enable processors', ); } if (this.processor) { await this.internalStopProcessor(); } const processorOptions = { kind: this.kind, track: this._mediaStreamTrack, // RN won't have or use AudioContext audioContext: this.audioContext as AudioContext, localTrack: this, }; this.log.debug(`setting up audio processor ${processor.name}`, this.logContext); await processor.init(processorOptions); if (this.stopCount !== stops) { await this.destroyUnusedProcessor(processor); throw new TrackInvalidError('track was stopped while its processor was being set'); } this.processor = processor; if (processor.processedTrack) { await this.sender?.replaceTrack(processor.processedTrack); // the stop destroyed the processor if (this.stopCount !== stops) { throw new TrackInvalidError('track was stopped while its processor was being set'); } this.observeProcessedTrack(processor.processedTrack); } this.emit(TrackEvent.TrackProcessorUpdate, this.processor); } finally { unlock(); } } protected override async onMediaStreamTrackChanged(): Promise { // the processor rebuilds its graph around the new track, so the krisp listeners have to // follow it onto whatever it hands back this.observeProcessedTrack(this.processor?.processedTrack); } protected async internalStopProcessor(keepElement = true) { this.observeProcessedTrack(undefined); await super.internalStopProcessor(keepElement); } stop() { this.observeProcessedTrack(undefined); super.stop(); } /** * @internal * @experimental * * A processor's nodes (`AudioWorkletNode`, ...) can't move between contexts, so the processor * is rebuilt whenever the context changes and torn down when there is none left to run in. */ async setAudioContext(audioContext: AudioContext | undefined) { if (this.audioContext === audioContext) { return; } const unlock = await this.trackChangeLock.lock(); const stops = this.stopCount; try { this.audioContext = audioContext; if (!this.processor || isReactNative()) { return; } if (!audioContext) { this.log.debug('stopping audio processor, audio context has been removed', this.logContext); await this.internalStopProcessor(); return; } this.log.debug('restarting audio processor on new audio context', this.logContext); const processor = this.processor; await processor.restart({ kind: this.kind, track: this._mediaStreamTrack, audioContext, localTrack: this, }); // the stop destroyed the processor if (this.stopCount !== stops) { return; } if (processor.processedTrack) { await this.sender?.replaceTrack(processor.processedTrack); this.observeProcessedTrack(processor.processedTrack); } this.emit(TrackEvent.TrackProcessorUpdate, this.processor); } finally { unlock(); } } async getSenderStats(): Promise { if (!this.sender?.getStats) { return undefined; } const stats = await this.sender.getStats(); let audioStats: AudioSenderStats | undefined; stats.forEach((v) => { if (v.type === 'outbound-rtp') { audioStats = { type: 'audio', streamId: v.id, packetsSent: v.packetsSent, bytesSent: v.bytesSent, timestamp: v.timestamp, }; // loss, jitter and RTT are only known from what the remote reports back, // the same way the video sender picks them up const remote = stats.get(v.remoteId); if (remote) { audioStats.packetsLost = remote.packetsLost; audioStats.jitter = remote.jitter; audioStats.roundTripTime = remote.roundTripTime; } } }); return audioStats; } async checkForSilence() { const trackIsSilent = await detectSilence(this); if (trackIsSilent) { if (!this.isMuted) { this.log.debug('silence detected on local audio track', this.logContext); } this.emit(TrackEvent.AudioSilenceDetected); } return trackIsSilent; } }