/** * melspectrogram.ts, the wake-word front end's first stage, computed in code. * * The pipeline behind the pinned classifier is * * audio -> melspectrogram -> speech-embedding backbone -> classifier * * and openWakeWord distributes the melspectrogram stage as a downloadable model * file. It does not need to be one: the stage is a fixed DSP graph, a short-time * Fourier transform, a mel filterbank, and a decibel conversion, with **no * learned parameters at all**. Computing it here removes a runtime download, a * checksum to manage, and an inference session from the hot loop. * * WHY THESE EXACT CONSTANTS * * The classifier was trained against openWakeWord's front end. If this stage * drifts, every measured recall and false-accept number in * `docs/wake-word-model.md` silently stops describing the running detector. So * the constants below were not chosen, and were not taken from any library's * defaults or from any reference implementation's prose: they were **recovered * numerically from openWakeWord's own `melspectrogram.onnx` weights**, which is * a `torchlibrosa` export whose graph is * * Conv(real basis) / Conv(imag basis) -> real^2 + imag^2 -> MatMul(melW) * -> clip(1e-10, inf) -> log -> *10 -> /ln(10) -> -0 * -> clip(min = globalMax - 80) * * Fitting each stage against those initializers gives, to float32 precision: * * - window periodic Hann of length 400, zero-padded and CENTRED in the * 512-point frame (56 zeros each side). Recovered by reading * row k=0 of `0.stft.conv_real.weight`, which is the window * itself because cos(0) = 1. Max abs deviation 2.8e-8. * - Fourier basis conv_real[k][n] = w[n]*cos(2*pi*k*n/512), * conv_imag[k][n] = -w[n]*sin(2*pi*k*n/512), not time-reversed. * Max abs deviation 5.6e-8 over all 257x512 taps. * - hop 160 samples, from the Conv `strides` attribute. * - padding none. The Conv carries `pads=[0,0]`, so this is * `center=False` framing, NOT librosa's centred default. * - filterbank 32 mel bands, Slaney mel scale, Slaney area normalisation, * fmin 60 Hz, fmax 3800 Hz. Max abs deviation from the pinned * `1.melW` matrix 8.1e-10 against a peak weight of 1.4e-2. * - decibels power (not magnitude) spectrogram, amin 1e-10, ref 1.0, * top_db 80, and the top_db floor uses the maximum over the * WHOLE call's output, reproduced in {@link melFrames}. * * Those deviations are the parity evidence for the DSP constants themselves; * end-to-end score parity against the real front end is asserted by * `test/wake-word-front-end-parity.test.ts` against committed reference frames. * * Everything here is plain arithmetic on typed arrays, so it runs unchanged in * a daemon child process and in a browser. */ /** Sample rate the whole wake pipeline assumes, in Hz. */ export declare const WAKE_SAMPLE_RATE = 16000; /** FFT size, in samples. */ export declare const WAKE_MEL_N_FFT = 512; /** Hop between consecutive frames, in samples (10 ms at 16 kHz). */ export declare const WAKE_MEL_HOP = 160; /** Non-zero window length, in samples (25 ms at 16 kHz). */ export declare const WAKE_MEL_WIN_LENGTH = 400; /** Number of mel bands the embedding backbone expects. */ export declare const WAKE_MEL_BINS = 32; /** Lowest mel filter edge, in Hz. */ export declare const WAKE_MEL_FMIN = 60; /** Highest mel filter edge, in Hz. */ export declare const WAKE_MEL_FMAX = 3800; /** Power floor before the log, matching librosa's `amin`. */ export declare const WAKE_MEL_AMIN = 1e-10; /** Dynamic-range floor below the per-call peak, matching librosa's `top_db`. */ export declare const WAKE_MEL_TOP_DB = 80; /** Number of one-sided FFT bins: 512/2 + 1. */ export declare const WAKE_MEL_FFT_BINS: number; /** * Minimum samples needed to produce one frame. Framing is `center=False`, so a * frame needs a full window and nothing is padded. */ export declare const WAKE_MEL_MIN_SAMPLES = 512; /** * How many frames {@link melFrames} produces for an input of `sampleCount` * samples. Zero when the input is shorter than one frame. */ export declare function melFrameCount(sampleCount: number): number; /** * The pinned filterbank, exposed so a parity test can compare it against * openWakeWord's `1.melW` initializer directly rather than only end-to-end. * Row-major `[fftBin * 32 + melBand]`. */ export declare function melFilterbank(): Float64Array; /** * The pinned analysis window, exposed for the same reason as * {@link melFilterbank}: it is checkable against row 0 of the reference graph's * real convolution weights. */ export declare function analysisWindow(): Float64Array; /** * Scratch buffers for one melspectrogram call. Reused across calls so the * streaming loop allocates nothing per frame. */ interface MelScratch { readonly re: Float64Array; readonly im: Float64Array; readonly power: Float64Array; } /** * Compute the log-mel spectrogram of `samples`, returning `frames * 32` values * row-major (frame-major). * * `samples` are raw int16 magnitudes as floats, the same scaling openWakeWord * feeds its front end, i.e. NOT normalised to [-1, 1]. Feeding normalised audio * shifts every value by a constant 90.3 dB and the classifier's scores become * meaningless, so the scale is part of the contract, not a detail. * * The decibel floor is `peak - 80` where `peak` is the maximum over everything * this call produced. That makes the result depend on the call's framing, which * is why the streaming pipeline always hands over a fixed-size window rather * than whatever audio happens to be buffered. */ export declare function melFrames(samples: Float32Array | Float64Array, scratch?: MelScratch): Float32Array; /** * openWakeWord's own rescaling of the melspectrogram before the embedding * backbone: `value / 10 + 2`. Its source comments this as bringing the ONNX * melspectrogram in line with the original TensorFlow implementation from * `tfhub google/speech_embedding/1`. The embedding model was trained on the * rescaled values, so this is part of the front end and not a preference. * * Applied in place; returns the same array for chaining. */ export declare function applyEmbeddingScaling(frames: Float32Array): Float32Array; /** Allocate a private scratch set, for a caller running several streams at once. */ export declare function createMelScratch(): MelScratch; export type { MelScratch }; //# sourceMappingURL=melspectrogram.d.ts.map