/** * A browser-dictation SPEECH GATE for Whisper ("Smart AI") mode. * * The problem it solves: Whisper/gpt-4o-transcribe HALLUCINATES words on silence, * background noise, or keyboard clicks ("I don't know", "you", "Thank you.") — so the * amplitude VAD would record near-nothing, upload it, and the hallucination got sent as * a real turn ("I'm not talking and it started working"). Browser SpeechRecognition * (Web Speech) does NOT hallucinate on non-speech — it simply returns nothing. So we run * it alongside the Whisper recorder purely to answer two questions in real time: * * 1. "Show me the words as I speak" — live interim text (always-visible dictation). * 2. "Did the user actually say real words this turn?" — the gate; if not, the turn is * discarded and never sent to the model (no phantom turn, no "Working"). * * Feature-detected: returns null when SpeechRecognition is unavailable (iOS Safari), where * the caller falls back to the amplitude VAD + the noise-transcript filter. */ import { type MicErrorDetail } from "./mic-retry.js"; /** * Callbacks for the gate's CONTROL-LISTENER mode (#744). * * When the main Whisper recorder is idle the gate's recognizer has nothing to do, so it is * repurposed as the always-on control listener rather than sitting stopped while a SECOND * `SpeechRecognition` object handles that role. The gate and the control listener are already * mutually exclusive by design (`shouldRunControlListener`), so this is purely a dispatch * change: one device owner, two consumers served in turns. */ export interface ControlOpts { /** A result arrived — same contract as VoiceStt's `onResult`. */ onResult: (text: string, isFinal: boolean) => void; /** The recognizer could not have the microphone. Same reportability rules as the gate's own * `onMicError`: only codes that are worth a durable row (#425). */ onError: (code: string, detail: MicErrorDetail) => void; /** The recognizer ended (terminal or denied). The caller re-arms if it still wants to. */ onEnd: () => void; } export interface SpeechGate { /** Begin listening (idempotent). */ start(): void; /** Stop listening (idempotent). Keeps the instance for a later start(). */ stop(): void; /** True once at least one real (non-noise) word was heard since the last reset. */ heardSpeech(): boolean; /** True once the recognizer has produced any event at all, EVER — including `onend`, which a * recognizer refused the microphone still fires. So this answers "has this engine ever run", * not "is it hearing now", and no verdict may use it (#535). Diagnostic only. */ isAlive(): boolean; /** * Does this gate have the microphone FOR THE CURRENT TURN? * * The question `planTurnClose` was asking `isAlive` and getting the wrong answer to. Set by * `onaudiostart` (the UA reports capture has begun) or by any result arriving; cleared by * `reset()` at every turn boundary and by any non-benign recognizer error — `audio-capture`, * the code the two background recognizers produce fighting over one device, and the one on ten * `client:voice-gate` rows the morning #535 was filed. * * False is the SAFE value: a gate that is not hearing abstains, and an abstaining gate cannot * destroy a turn. A browser that never fires `onaudiostart` therefore loses the veto and keeps * everything else. */ isHearing(): boolean; /** Did any transcript at all arrive this turn, accepted or not? Diagnostic only — it is what * distinguishes "the gate had no device" from "the gate had words and `acceptSpeech` or the * noise filter declined them", which #535 could otherwise only settle by watching the screen. */ sawWords(): boolean; /** Clear the PER-TURN facts — heard, hearing, saw-words and any buffered interim (call at the * start of each turn). `isAlive` is session-lifetime and deliberately survives. */ reset(): void; /** * Run the gate's recognizer as the always-on CONTROL LISTENER (#744). * * The gate and the control listener are mutually exclusive by design * (`shouldRunControlListener` in `convo.ts`): the gate runs during a Whisper turn, the control * listener runs when the main mic is idle. This method repurposes the SAME `SpeechRecognition` * object for both roles, removing the second start/stop/re-arm cycle that produces the * `audio-capture` rows at turn boundaries. * * Results are dispatched to `opts.onResult` instead of the dictation handlers. The * recognizer re-arms on `onend` exactly as the old `VoiceStt("browser")` control listener * did, and delegates error decisions to `opts.onError` / `opts.onEnd`. * * Idempotent: calling again with different opts while already in control mode updates the * callbacks (the language may have changed) without restarting the recognizer. */ startAsControl(opts: ControlOpts): void; /** Exit control mode and stop the recognizer (idempotent). */ stopControl(): void; } export interface SpeechGateOptions { lang?: string; /** * Live text as the user speaks — the always-visible dictation. * * TWO arguments, because two consumers want two different things and conflating them is a * regression either way (#458): * - `text` is the whole UTTERANCE so far ({@link reduceHeard}), which is what the bubble * shows and what `heard` is stamped from. Before #458 it was the phrase, so a pause erased * everything said before it. * - `phrase` is just what changed in THIS event — the recogniser's own unit. It is what a * command scanner must see: a single-word command only fires when it IS the whole utterance * ({@link phraseMatchesTranscript}), so handing an accumulator to the matcher would make * "run the tests" + "mute" unmatchable and delete mute-during-capture (#228, ADR 0001 M1). */ onInterim: (text: string, phrase: string) => void; /** Fired the first time real words are heard this turn (→ the turn is legit). */ onSpeech?: () => void; /** * Could the words arriving RIGHT NOW belong to the user? Default: yes. * * The heard-speech flag is what `speechVerdict` trusts to discard a silent clip, and it * used to latch on ANY recognised words — including the agent's own TTS coming back through * the speakers, and anything picked up while a reply was in flight. So the gate would vouch * for a turn in which the user had not spoken at all, the clip was uploaded, and Whisper * returned a term from its own vocabulary prompt (#332: two turns reading "Coder Lead", the * agent's name, with the user disengaged and the mic still open). * * The caller already computes exactly this predicate for the live-transcript path * (`shouldIgnoreResult` — echo tail plus paused plus muted). Passing it here is what makes * the veto and the display agree about whose voice they are hearing. */ acceptSpeech?: () => boolean; /** * The recognizer could not have the microphone (#425). * * `r.onerror` was an empty block. That silence had two costs: nobody could learn that a gate had * died, and `planNoiseRejection` treats a dead gate as "cannot vouch for this turn", so a * blocked gate quietly changes which marginal turns survive. Reported through a callback rather * than by importing the client here, so this module stays a pure browser adapter and the test * can observe the call without a network stub. * * Named `onDenied` and gated on `isMicPermissionDenied` until the #425 reopen, which is how the * two background recognizers ended up asymmetric: the control listener reported `audio-capture` * and the gate did not, so the one contention code produced by two recognizers fighting over one * device — the code actually seen in production — was invisible from exactly the recognizer that * competes with the recorder. Reporting is now {@link isReportableMicError} on both. */ onMicError?: (code: string, detail: MicErrorDetail) => void; } /** Is browser dictation available at all? (Used to decide whether to gate.) */ export declare function speechGateAvailable(): boolean; /** * Create a speech gate, or null when SpeechRecognition isn't available. The gate runs * continuously (auto-restarts on end while active) so it never stops watching mid-turn. */ export declare function createSpeechGate(opts: SpeechGateOptions): SpeechGate | null; //# sourceMappingURL=gate.d.ts.map