import { EventEmitter } from 'events'; import { WarningName } from './constants'; import { AlreadyStoppedError, DiagnosticError, InvalidOptionsError, } from './errors'; import { AudioContext, AudioContextUnsupportedError, enumerateDevices, EnumerateDevicesUnsupportedError, getDefaultDevices, getUserMedia, GetUserMediaUnsupportedError, } from './polyfills'; import { AudioRecorder } from './recorder/audio'; import { SubsetRequired, TimeMeasurement, VolumeStats } from './types'; import { InvalidityRecord, validateDeviceId, validateOptions, validateTime, } from './utils/optionValidation'; export declare interface AudioInputTest { /** * This event is emitted with a test report when the test ends. * @param event [[AudioInputTest.Events.End]] * @param report Summary of the test. * @private */ emit( event: AudioInputTest.Events.End, report: AudioInputTest.Report, ): boolean; /** * This event is emitted with a [[DiagnosticError]] when the test encounters * an error, fatal or not. * @param event [[AudioInputTest.Events.Error]] * @param error The [[DiagnosticError]] that was encountered. * @private */ emit( event: AudioInputTest.Events.Error, error: DiagnosticError, ): boolean; /** * This event is emitted with a volume level every * [[AudioInputTest.Options.volumeEventIntervalMs]] after the test starts succesfully. * @param event [[AudioInputTest.Events.Volume]] * @param value The current volume of the audio source. * @private */ emit( event: AudioInputTest.Events.Volume, value: number, ): boolean; /** * This event is emitted when the test encounters a non-fatal warning during * its run-time. * @param event [[AudioInputTest.Events.Warning]] * @param warning The warning that the test encountered. * @private */ emit( event: AudioInputTest.Events.Warning, warning: WarningName, ): boolean; /** * This event is emitted when the test clears a previously encountered * non-fatal warning during its run-time. * @param event [[AudioInputTest.Events.WarningCleared]] * @param warning The warning that the test encountered that should be cleared. * @private */ emit( event: AudioInputTest.Events.WarningCleared, warning: WarningName, ): boolean; /** * Raised upon completion of the test. * @param event [[AudioInputTest.Events.End]] * @param listener A callback that expects the following parameters: * An [[AudioInputTest.Report]] that summarizes the test. * @returns This [[AudioInputTest]] instance. * @event */ on( event: AudioInputTest.Events.End, listener: (report: AudioInputTest.Report) => any, ): this; /** * Raised by the test when encountering an error with a parameter of type * [[DiagnosticError]]. * @param event [[AudioInputTest.Events.Error]] * @param listener A callback that expects the following parameters: * A [[DiagnosticError]] that the test encountered. * @returns This [[AudioInputTest]] instance. * @event */ on( event: AudioInputTest.Events.Error, listener: (error: DiagnosticError) => any, ): this; /** * Raised by the test every [[Options.volumeEventIntervalMs]] amount of * milliseconds with a parameter of type `number` that represents the * current volume of the audio stream. * @param event [[AudioInputTest.Events.Volume]] * @param listener A callback that expects the following parameters: * A `number` that represents the audio source's current volume. * @returns This [[AudioInputTest]] instance. * @event */ on( event: AudioInputTest.Events.Volume, listener: (value: number) => any, ): this; /** * Raised by the test when the test encounters a non-fatal warning during * its run-time. * @param event [[AudioInputTest.Events.Warning]] * @param listener A callback that expects the following parameters: * A [[WarningName]]. * @returns This [[AudioInputTest]] instance. * @event */ on( event: AudioInputTest.Events.Warning, listener: (warningName: WarningName) => any, ): this; /** * Raised by the test when the test clears a previously encountered non-fatal * warning during its run-time. * @param event [[AudioInputTest.Events.WarningCleared]] * @param listener A callback that expects the following parameters: * A [[WarningName]] name. * @returns This [[AudioInputTest]] instance. * @event */ on( event: AudioInputTest.Events.WarningCleared, listener: (warningName: WarningName) => any, ): this; } /** * [[AudioInputTest]] class that parses options and starts an audio input device * test. * * Please see [[testAudioInputDevice]] for details and recommended practices. */ export class AudioInputTest extends EventEmitter { /** * Name of the test. */ static readonly testName: string = 'audio-input-test'; /** * Default options for the [[AudioInputTest]]. */ private static defaultOptions: AudioInputTest.InternalOptions = { audioContextFactory: AudioContext, audioRecorderFactory: AudioRecorder, debug: false, duration: Infinity, enableRecording: false, enumerateDevices, getUserMedia, volumeEventIntervalMs: 100, }; /** * Active warnings to keep track of. */ readonly activeWarnings: Set = new Set(); /** * An `AudioContext` to use for generating volume levels. */ private _audioContext: AudioContext | null = null; /** * An AudioRecorder object used to capture audio input during the test */ private _audioRecorder: AudioRecorder | null = null; /** * A function that will be assigned in `_startTest` that when run will clean * up the audio nodes created in the same function. */ private _cleanupAudio: (() => void) | null = null; /** * The default media devices when starting the test. */ private _defaultDevices: Partial> = {}; /** * A timestamp that is set when the test ends. */ private _endTime: number | null = null; /** * An array of any errors that occur during the run time of the test. */ private readonly _errors: DiagnosticError[] = []; /** * A `MediaStream` that is created from the input device. */ private _mediaStream: MediaStream | null = null; /** * Options that are passed to and set in the constructor for use during the * test. */ private _options: AudioInputTest.InternalOptions; /** * A timestamp that is set when the test starts after a successful call to getUserMedia. */ private _startTime: number | undefined; /** * Volume levels generated from the audio source during the run time of the * test. */ private readonly _volumeStats: VolumeStats = { timestamps: [], values: [], }; /** * The timeout that causes the volume event to loop; created by `setTimeout`. */ private _volumeTimeout: NodeJS.Timeout | null = null; /** * Initializes the `startTime` and `options`. * @param options Optional settings to pass to the test. */ constructor(options?: AudioInputTest.Options) { super(); this._options = { ...AudioInputTest.defaultOptions, ...options }; // We need to use a `setTimeout` here to prevent a race condition. // This allows event listeners to bind before the test starts. setTimeout(() => this._startTest()); } /** * Stop the currently running [[AudioInputTest]]. */ stop(): void { if (typeof this._endTime === 'number') { this._onWarning(new AlreadyStoppedError()); return; } this._endTime = Date.now(); const report: AudioInputTest.Report = { deviceId: this._options.deviceId || ( this._defaultDevices.audioinput && this._defaultDevices.audioinput.deviceId ), errors: this._errors, testName: AudioInputTest.testName, values: this._volumeStats.values, }; if (this._startTime) { report.testTiming = { duration: this._endTime - this._startTime, end: this._endTime, start: this._startTime, }; } const onEnd = () => { this._cleanup(); this.emit(AudioInputTest.Events.End, report); }; if (this._options.enableRecording && this._audioRecorder) { this._audioRecorder.stop().then(() => { report.recordingUrl = this._audioRecorder!.url; }).catch((ex: DiagnosticError) => { this._onError(ex); }).finally(onEnd); } else { onEnd(); } } /** * Clean up any instantiated objects (i.e. `AudioContext`, `MediaStreams`, * etc.). * Called by `.stop`. */ private _cleanup(): void { if (this._volumeTimeout) { clearTimeout(this._volumeTimeout); } if (this._cleanupAudio) { this._cleanupAudio(); } if (this._mediaStream) { this._mediaStream.getTracks().forEach( (track: MediaStreamTrack) => track.stop(), ); } if (this._audioContext) { this._audioContext.close(); } } /** * Helper function that should be called when an error occurs, recoverable * or not. * @param error */ private _onError(error: DiagnosticError): void { this._errors.push(error); this.emit(AudioInputTest.Events.Error, error); } /** * Called every `AudioInputTest._options.volumeEventIntervalMs` amount of * milliseconds, emits the volume passed to it as a `Events.Volume` event. * @param value the volume */ private _onVolume(value: number): void { const now = Date.now(); if (!this._volumeStats.max || value > this._volumeStats.max) { this._volumeStats.max = value; } this._volumeStats.values.push(value); this._volumeStats.timestamps.push(now); this.emit(AudioInputTest.Events.Volume, value); // Find the last 3 seconds worth of volume values. const startIndex = this._volumeStats.timestamps.findIndex( (timestamp: number) => now - timestamp <= 3000, ); // We want to do nothing at 1 and not 0 here because this guarantees that // there is at least one timestamp before the sample set. This means that // there are at least three seconds of samples. if (startIndex < 1) { return; } const samples = this._volumeStats.values.slice( startIndex > 0 ? startIndex : 0, ); // Calculate the standard deviation of the sample set. const sampleAverage = samples.reduce( (sample: number, partialSum: number) => sample + partialSum, 0, ) / samples.length; const diffSquared = samples.map( (sample: number) => Math.pow(sample - sampleAverage, 2), ); const stdDev = Math.sqrt(diffSquared.reduce( (sample: number, partialSum: number) => sample + partialSum, 0, ) / samples.length); // 255 is max volume value; 2.55 is 1% of max const isConstantAudio = stdDev <= 2.55; if (isConstantAudio && sampleAverage <= 2.55) { if (!this.activeWarnings.has(WarningName.LowAudioLevel)) { this.activeWarnings.add(WarningName.LowAudioLevel); this.emit(AudioInputTest.Events.Warning, WarningName.LowAudioLevel); } } else if (this.activeWarnings.has(WarningName.LowAudioLevel)) { this.activeWarnings.delete(WarningName.LowAudioLevel); this.emit(AudioInputTest.Events.WarningCleared, WarningName.LowAudioLevel); } } /** * Warning event handler. * @param warning */ private _onWarning(error: DiagnosticError): void { if (this._options.debug) { // tslint:disable-next-line no-console console.warn(error); } } /** * Entry point into the audio input device test. Uses the `MediaStream` that the * object was set up with, and performs a fourier transform on the audio data * using an `AnalyserNode`. The output of the fourier transform are the * relative amplitudes of the frequencies of the audio data. The average of * this data can then be used as an estimate as the average volume of the * entire volume source. * * @event Events.Volume */ private async _startTest(): Promise { try { // Try to validate all of the inputs before starting the test. // We perform this check here so if the validation throws, it gets handled // properly as a fatal-error and we still emit a report with that error. const invalidReasons: InvalidityRecord | undefined = await validateOptions(this._options, { deviceId: validateDeviceId, duration: validateTime, volumeEventIntervalMs: validateTime, }); if (invalidReasons) { throw new InvalidOptionsError(invalidReasons); } if (!this._options.getUserMedia) { throw GetUserMediaUnsupportedError; } this._mediaStream = await this._options.getUserMedia({ audio: { deviceId: this._options.deviceId }, }); if (!this._options.audioContextFactory) { throw AudioContextUnsupportedError; } // We need to initialize AudioContext and MediaRecorder right after calling gUM // and before enumerateDevices. Certain browsers and headsets (Safari, AirPods) // loses the "user action" after enumerating devices. this._audioContext = new this._options.audioContextFactory(); if (this._options.enableRecording) { this._audioRecorder = new this._options.audioRecorderFactory!({ audioContext: this._audioContext, stream: this._mediaStream, }); } if (!this._options.enumerateDevices) { throw EnumerateDevicesUnsupportedError; } this._defaultDevices = getDefaultDevices( await this._options.enumerateDevices(), ); // Only starts the timer after successfully getting devices this._startTime = Date.now(); const analyser: AnalyserNode = this._audioContext.createAnalyser(); analyser.smoothingTimeConstant = 0.4; analyser.fftSize = 64; const microphone: MediaStreamAudioSourceNode = this._audioContext.createMediaStreamSource(this._mediaStream); microphone.connect(analyser); this._cleanupAudio = (): void => { analyser.disconnect(); microphone.disconnect(); }; const frequencyDataBytes: Uint8Array = new Uint8Array(analyser.frequencyBinCount); // This function runs every `this._options.reportRate` ms and emits the // current volume of the `MediaStream`. const volumeEvent: () => void = (): void => { if (this._endTime) { return; } analyser.getByteFrequencyData(frequencyDataBytes); const volume: number = frequencyDataBytes.reduce( (sum: number, val: number) => sum + val, 0, ) / frequencyDataBytes.length; this._onVolume(volume); if (Date.now() - this._startTime! > this._options.duration) { this.stop(); } else { this._volumeTimeout = setTimeout( volumeEvent, this._options.volumeEventIntervalMs, ); } }; this._volumeTimeout = setTimeout( volumeEvent, this._options.volumeEventIntervalMs, ); } catch (error) { if (error instanceof DiagnosticError) { // There is some other fatal error. this._onError(error); } else if ( typeof DOMException !== 'undefined' && error instanceof DOMException ) { this._onError(new DiagnosticError( error, 'A `DOMException` has occurred.', )); } else if ( typeof DOMError !== 'undefined' && error instanceof DOMError ) { this._onError(new DiagnosticError( error, 'A `DOMError` has occurred.', )); } else if ( typeof Error !== 'undefined' && error instanceof Error ) { this._onError(new DiagnosticError( error, 'An error has occurred.', )); } else { this._onError(new DiagnosticError( undefined, 'Unknown error occurred.', )); this._onWarning(error); } this.stop(); } } } export namespace AudioInputTest { /** * Possible events that an [[AudioInputTest]] might emit. See [[AudioInputTest.on]]. */ export enum Events { End = 'end', Error = 'error', Volume = 'volume', Warning = 'warning', WarningCleared = 'warning-cleared', } /** * Represents the report generated from an [[AudioInputTest]]. */ export interface Report { /** * The device ID used to get a MediaStream from using [getUserMedia](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia). */ deviceId: MediaTrackConstraintSet['deviceId']; /** * Any errors that occurred during the test. */ errors: DiagnosticError[]; /** * If [[AudioInputTest.Options.enableRecording]] is set to true, * `recordingUrl` will be available in the report which can be used to playback captured audio from the microphone. * Your application should revoke this URL if it is no longer needed. * * Example: * * ```ts * const audioInputTest: AudioInputTest = testAudioInputDevice({ enableRecording: true }); * audioInputTest.on(AudioInputTest.Events.End, (report: AudioInputTest.Report) => { * const audioEl = new Audio(); * audioEl.src = report.recordingUrl; * audioEl.play(); * * // Revoke the url if no longer needed * URL.revokeObjectURL(report.recordingUrl); * }); * ``` */ recordingUrl?: string; /** * The name of the test. */ testName: typeof AudioInputTest.testName; /** * Time measurements of test run time. */ testTiming?: TimeMeasurement; /** * The volume levels emitted by the test during its run-time. */ values: number[]; } /** * Options passed to [[AudioInputTest]] constructor. */ export interface Options { /** * AudioContext mock to be used during the test. * @private */ audioContextFactory?: typeof window.AudioContext; /** * AudioRecorder mock to be used during the test. * @private */ audioRecorderFactory?: typeof AudioRecorder; /** * Whether or not to log debug statements to the console. * @private */ debug?: boolean; /** * The device ID to try to get a MediaStream from using [getUserMedia](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia). */ deviceId?: MediaTrackConstraintSet['deviceId']; /** * Duration of time to run the test in ms. * @default Infinity */ duration?: number; /** * Whether the test should record the audio input or not. * If set to true, [[AudioInputTest.Report.recordingUrl]] will be available for audio playback. * Your application should revoke this URL if it is no longer needed. * * Example: * * ```ts * const audioInputTest: AudioInputTest = testAudioInputDevice({ enableRecording: true }); * audioInputTest.on(AudioInputTest.Events.End, (report: AudioInputTest.Report) => { * const audioEl = new Audio(); * audioEl.src = report.recordingUrl; * audioEl.play(); * * // Revoke the url if no longer needed * URL.revokeObjectURL(report.recordingUrl); * }); * ``` * @default false */ enableRecording?: boolean; /** * Used to mock the call to `enumerateDevices`. * @private */ enumerateDevices?: typeof navigator.mediaDevices.enumerateDevices; /** * Used to mock calls to `getUserMedia`. * @private */ getUserMedia?: typeof window.navigator.mediaDevices.getUserMedia; /** * The interval between emissions of volume events in milliseconds. * @default 100 */ volumeEventIntervalMs?: number; } /** * Option typing after initialization, so we can have type guarantees. * @private */ export type InternalOptions = SubsetRequired; } /** * [[AudioInputTest]] tests audio input capabilities. It serves to help diagnose * potential audio device issues that would prevent audio from being recognized * in a WebRTC call. * * --- * * The [[AudioInputTest]] class is an `EventEmitter` (please see [[AudioInputTest.on]] for * events and their details) and helps to diagnose issues by capturing user * audio and emitting the volume levels detected in that media. * ```ts * import { AudioInputTest, testAudioInputDevice } from '@twilio/rtc-diagnostics'; * const options: AudioInputTest.Options = { ... }; * // `options` may be left `undefined` to use default option values * const audioInputTest: AudioInputTest = testAudioInputDevice(options); * ``` * Applications can use the volume events emitted by the test to update their UI * to show to the user whether or not their media was captured successfully. * ```ts * audioInputTest.on(AudioInputTest.Events.Volume, (volume: number) => { * ui.updateVolume(volume); // Update your UI with the volume value here. * }); * ``` * The test can be normally stopped two ways: allowing the test to time out and * stopping the test manually. * * To end the test manually, the application can ask the end-user to confirm * that the volume levels it emits are what the end-user expects. If so, the * application can call the [[AudioInputTest.stop]] method with `true`. Otherwise, * if the audio values are not expected, the application can call * [[AudioInputTest.stop]] with `false`. * ```ts * // The UI should indicate that if the volume values are what the user * // expects, they can click this button to pass and stop the test... * const volumeCorrectButton = ...; * volumeCorrectButton.addEventListener('click', () => { * audioInputTest.stop(true); * }); * * // ...otherwise, if the volume levels are not what they expect, they can * // click this. * const volumeIncorrectButton = ...; * volumeIncorrectButton.addEventListener('click', () => { * audioInputTest.stop(false); * }); * ``` * Calling [[AudioInputTest.stop]] will immediately end the test. * * --- * * The [[AudioInputTest]] object will always emit a [[AudioInputTest.Report]] with the * [[AudioInputTest.Events.End]] event, regardless of the occurrence of errors during * the runtime of the test. * * Fatal errors will immediately end the test and emit a report such that the * value of [[AudioInputTest.Report.errors]] will contain the fatal error. * * Non-fatal errors will not end the test, but will be included in the value of * [[AudioInputTest.Report.errors]] upon completion of the test. * * --- * * Note: In Firefox, `deviceId` will be ignored, and instead the user will get a * browser pop-up where they can select the device they want to use. This is * unavoidable as it is Firefox's implementation of `getUserMedia()`. * * In most browsers, such as Chrome and Safari, when `getUserMedia()` is called, * a prompt will ask the user for broad microphone-access permissions. Then, the * parameters passed to `getUserMedia()` will determine the device that is * captured. * * Firefox differs in that the prompt will ask for a specific input device. * Regardless of the parameters passed to `getUserMedia()`, the device * selected in that prompt will be captured. If the user opts to have the * browser "Remember this selection" within the prompt, the device that was * selected will be captured by every future `getUserMedia()` call as well. * This selection will persist even through changes in the system OS, i.e. when * default devices are changed. In order to change the device, the user has to * revoke the webpage's microphone-access permissions for the prompt to show * again. * * Please see this link for more information on microphone access in Firefox: * https://support.mozilla.org/en-US/kb/how-manage-your-camera-and-microphone-permissions * * --- * * The function [[testAudioInputDevice]] serves as a factory function that accepts * [[AudioInputTest.Options]] as its only parameter and will instantiate an * [[AudioInputTest]] object with those options. * ```ts * import { AudioInputTest, testAudioInputDevice } from '@twilio/rtc-diagnostics'; * const options: AudioInputTest.Options = { ... }; * const audioInputTest: AudioInputTest = testAudioInputDevice(options); * ``` * * @param options Options to pass to the [[AudioInputTest]] constructor. */ export function testAudioInputDevice( options?: AudioInputTest.Options, ): AudioInputTest { return new AudioInputTest(options); }