import KrispSDK from './krispai'; import type { IAudioFilterNode, IKrispSDK, ISDKPartialOptions, } from './krispai'; import { packageName, packageVersion } from './version'; import { promiseWithResolvers } from './withResolvers'; import { simd } from 'wasm-feature-detect'; import type { Tracer } from './tracer'; const MODEL_FILENAME = 'krisp-nc-o-med-v7.kef'; /** * Options to pass to the NoiseCancellation instance. */ export type NoiseCancellationOptions = { /** * The base path to load the models from. * You can override this if you want to host the models yourself. * @default `https://unpkg.com/@stream-io/audio-filters-web@${packageVersion}/src/krispai/models`. */ basePath?: string; /** * When Krisp SDK detects buffer overflow, it will disable the filter and * wait for this timeout before enabling it again. * Defaults to 15000 ms. */ restoreTimeoutMs?: number; /** * The number of attempts to restore the filter after a buffer overflow. * Defaults to 3. */ restoreAttempts?: number; /** * Optional Krisp SDK parameters. */ krispSDKParams?: ISDKPartialOptions['params']; }; /** * An interface for the NoiseCancellation implementation. * Provided for easier unit testing. */ export interface INoiseCancellation { isSupported: () => boolean | Promise; init: (options?: { tracer?: Tracer }) => Promise; isEnabled: () => Promise; canAutoEnable?: () => Promise; enable: () => Promise; disable: () => Promise; dispose: () => Promise; resume: () => void; setSuppressionLevel: (level: number) => void; toFilter: () => (mediaStream: MediaStream) => { output: MediaStream; }; on: ( event: E, callback: T, ) => () => void; off: (event: E, callback: T) => void; } /** * A list of events one can subscribe to. */ export type Events = { /** * Fires when Noise Cancellation state changes. * * @param enabled true when enabled, false otherwise. */ change: (enabled: boolean) => void; }; /** * A wrapper around the Krisp.AI SDK. */ export class NoiseCancellation implements INoiseCancellation { private sdk?: IKrispSDK; private filterNode?: IAudioFilterNode; private audioContext?: AudioContext; private restoreTimeoutId?: number; private tracer?: Tracer; private readonly initializing: Promise; private readonly resolveInitialized!: () => void; private readonly basePath: string; private readonly restoreTimeoutMs: number; private readonly restoreAttempts: number; private readonly krispSDKParams?: ISDKPartialOptions['params']; private readonly listeners: Partial>> = {}; /** * Constructs a new instance. */ constructor({ basePath = `https://unpkg.com/${packageName}@${packageVersion}/src/krispai/models`, restoreTimeoutMs = 15000, restoreAttempts = 3, krispSDKParams, }: NoiseCancellationOptions = {}) { const { promise, resolve } = promiseWithResolvers(); this.initializing = promise; this.resolveInitialized = resolve; this.basePath = basePath; this.restoreTimeoutMs = restoreTimeoutMs; this.restoreAttempts = restoreAttempts; this.krispSDKParams = krispSDKParams; } /** * Checks if the noise cancellation is supported on this platform. * Make sure you call this method before trying to enable the noise cancellation. */ isSupported = () => { if (!KrispSDK.isSupported()) { return false; } else { return simd(); } }; /** * Initializes the KrispAI SDK. * * Will throw in case the noise cancellation is not supported on this platform * or if the SDK is already initialized. */ init = async (options: { tracer?: Tracer } = {}) => { if (!(await this.isSupported())) { throw new Error('NoiseCancellation is not supported on this platform'); } if (this.sdk) { throw new Error('NoiseCancellation is already initialized'); } const sdk = new KrispSDK({ params: { debugLogs: false, logProcessStats: false, useSharedArrayBuffer: false, models: { // https://sdk-docs.krisp.ai/docs/krisp-audio-sdk-model-selection-guide modelNC: `${this.basePath}/${MODEL_FILENAME}`, }, ...this.krispSDKParams, }, }); await sdk.init(); this.sdk = sdk; this.tracer = options.tracer; const audioContext = new AudioContext(); this.audioContext = audioContext; this.tracer?.trace( 'noiseCancellation.audioContextState', audioContext.state, ); this.audioContext.addEventListener('statechange', () => { this.tracer?.trace( 'noiseCancellation.audioContextState', audioContext.state, ); }); // AudioContext requires user interaction to start: // https://developer.chrome.com/blog/autoplay/#webaudio const resume = () => { this.resume(); document.removeEventListener('click', resume); }; if (this.audioContext.state === 'suspended') { document.addEventListener('click', resume); } const filterNode = await sdk.createNoiseFilter( this.audioContext, () => { this.tracer?.trace('noiseCancellation.started', 'true'); this.resolveInitialized(); }, () => document.removeEventListener('click', resume), ); filterNode.addEventListener('buffer_overflow', this.handleBufferOverflow); this.filterNode = filterNode; return this.initializing; }; /** * Checks if the noise cancellation is enabled. */ isEnabled = async () => { if (!this.filterNode) return false; return this.filterNode.isEnabled(); }; /** * Enables the noise cancellation. */ enable = async () => { if (!this.filterNode) return; await this.initializing; this.filterNode.enable(); this.dispatch('change', true); }; /** * Disables the noise cancellation. */ disable = async () => { if (!this.filterNode) return; await this.initializing; this.filterNode.disable(); this.dispatch('change', false); }; /** * Disposes the instance and releases all resources. */ dispose = async () => { window.clearTimeout(this.restoreTimeoutId); if (this.audioContext && this.audioContext.state !== 'closed') { await this.audioContext.close().catch((err) => { console.warn('Failed to close the audio context', err); }); this.audioContext = undefined; } if (this.filterNode) { await this.disable(); this.filterNode.removeEventListener( 'buffer_overflow', this.handleBufferOverflow, ); this.filterNode.dispose(); this.filterNode = undefined; } if (this.sdk) { this.sdk.dispose(); this.sdk = undefined; } }; /** * Sets the noise suppression level (0-100). * * @param level 0 for no suppression, 100 for maximum suppression. */ setSuppressionLevel = (level: number) => { // @ts-expect-error not yet in the types, but exists in the implementation if (!this.filterNode || !this.filterNode.setNoiseSuppressionLevel) { throw new Error( 'NoiseCancellation is not initialized with a filter node that supports noise suppression level', ); } if (level < 0 || level > 100) { throw new Error('NoiseCancellation level must be between 0 and 100'); } // @ts-expect-error not yet in the types, but exists in the implementation this.filterNode.setNoiseSuppressionLevel(level); }; /** * A utility method convenient for our Microphone filters API. */ toFilter = () => (mediaStream: MediaStream) => { if (!this.filterNode || !this.audioContext) { throw new Error('NoiseCancellation is not initialized'); } const [audioTrack] = mediaStream.getAudioTracks(); if (!audioTrack) throw new Error('No audio track found in the stream'); const source = this.audioContext.createMediaStreamSource(mediaStream); const destination = this.audioContext.createMediaStreamDestination(); destination.channelCount = audioTrack.getSettings().channelCount ?? 1; source.connect(this.filterNode).connect(destination); // When filter is started, user's microphone media stream is active. // That means that most probably we can resume audio context without // any autoplay policy limitations. this.resume(); return { output: destination.stream }; }; resume = () => { // resume if still suspended if (!this.audioContext) return; const state = this.audioContext.state; if (state === 'suspended' || state === 'interrupted') { const tag = 'audioContextResumeNC'; this.audioContext.resume().then( () => this.tracer?.trace(tag, this.audioContext?.state), (err) => { const data = [this.audioContext?.state, err?.message]; this.tracer?.trace(`${tag}Error`, data); console.warn( 'Failed to resume the audio context. Noise Cancellation may not work correctly', err, ); }, ); } }; /** * Registers the given callback to the event type; * * @param event the event to listen. * @param callback the callback to call. */ on = (event: E, callback: T) => { (this.listeners[event] ??= [] as T[]).push(callback); return () => { this.off(event, callback); }; }; /** * Unregisters the given callback for the event type. * * @param event the event. * @param callback the callback to unregister. */ off = (event: E, callback: T) => { const listeners = this.listeners[event] || []; this.listeners[event] = listeners.filter((cb) => cb !== callback); }; /** * Dispatches a new event payload for the given event type. * * @param event the event. * @param payload the payload. */ private dispatch = [0]>( event: E, payload: P, ) => { const listeners = this.listeners[event] || []; for (const listener of listeners) { listener(payload); } }; /** * Handles the buffer overflow event. * Disables the filter and waits for the restore timeout before enabling it again. * * Based on: https://sdk-docs.krisp.ai/docs/getting-started-js#system-overload-handling */ private handleBufferOverflow = ( // extending the Event type to include the data property as it is not yet // in the types but exists in the implementation e: Event & { data?: { overflowCount: number } }, ) => { const count = (e && e.data && e.data.overflowCount) ?? 0; this.tracer?.trace('noiseCancellation.bufferOverflowCount', String(count)); window.clearTimeout(this.restoreTimeoutId); this.disable().catch((err) => console.error('Failed to disable noise cancellation ', err), ); if (count < this.restoreAttempts) { this.restoreTimeoutId = window.setTimeout(() => { this.enable().catch((err) => console.error('Failed to enable noise cancellation ', err), ); }, this.restoreTimeoutMs); } }; }