/** Speech-to-Text abstraction — browser Web Speech API or OpenAI Whisper */ import { type GateEvidence } from "./machine.js"; import { type MicErrorDetail } from "./mic-retry.js"; import { type ClipVerdict, type LevelSnapshot } from "./vad.js"; /** OpenAI's real-time transcription model — replaces the legacy batch `whisper-1`. * Same upload endpoint, far better accuracy/latency, no verbose_json (we use json). */ export declare const DEFAULT_STT_MODEL = "gpt-4o-transcribe"; /** * `no_speech_prob` threshold for discarding a whisper-1 transcription as silence. * OpenAI's documented threshold for "probably no speech" is 0.6. When the model * returns a value above this, it has low confidence that real speech was present — * treat the transcript as a hallucination and emit "no-speech" instead. * Only applies to the non-streaming path (whisper-1 with response_format=verbose_json). */ export declare const NO_SPEECH_PROB_THRESHOLD = 0.6; /** * Can this model tell us it heard no speech? (#511 criterion 1.) * * Only `whisper-1`. `no_speech_prob` rides on `response_format=verbose_json`, which the * `gpt-4o-transcribe` family does not support — they take `json`, and we stream them. #490 shipped * this layer as one of three defences against phantom turns and it landed in the `else` of a * `const streaming = this.model !== "whisper-1"`, forty lines from the decision, where it reads as * a streaming-transport detail rather than what it is: **a capability only the legacy model has.** * The consequence went unnoticed for a day — the default model is `gpt-4o-transcribe`, so the * model-side gate protected nobody, and phantoms were still reaching Whisper 13 hours after #490 * deployed. Naming the capability is what stops "we have a confidence gate" being said of a * request that cannot carry one. * * The streaming model's phantom defence is therefore entirely ours, in three layers that DO run on * it: the adaptive energy gate before upload (`planClipGate`), `isNoiseTranscript` on the way to * the agent, and the low-energy suppression of the vocabulary bias prompt — without which silence * does not come back empty but comes back as a fluent sentence assembled from our own term list. * Both phantoms dated in #511 were caught by the second of those; neither became a user turn. * * NOT added here: `include[]=logprobs`, which is the one model-side confidence signal * `gpt-4o-transcribe` does expose. It would be a new, blind discard layer on the ONLY transcription * path anyone uses, tuned against no data, deployed on push — while the live complaint being worked * is speech going MISSING. Wrong direction to guess in. It is worth doing against the peakLevel / * onsetFloor rows #510 now writes, not before them. */ export declare function supportsNoSpeechProb(model: string): boolean; /** * Deadlines on a transcription request (#421). * * There were none — no `signal`, no `AbortController` anywhere in `voice/`, and none on the server * proxy either — so a request that stalled produced neither a `catch` nor a `!res.ok`, and the * whole failure apparatus in `use-voice.ts` hangs off `onError`. A promise that never settles * reaches none of it: the bubble sat on "Transcribing…" indefinitely and, until this change, held * the composer with it. Reload was the only escape. * * Two deadlines because there are two ways to stall. `FIRST_BYTE` covers a request that never * answers; `STREAM` covers one that answers and then stops mid-SSE, which the first cannot see. * Both start generous on purpose — a slow mobile network that is genuinely working must not be * failed, and a retry spends the user's own OpenAI credit. Tighten only against timeout rows in the * durable log. */ export declare const TRANSCRIBE_FIRST_BYTE_MS = 20000; export declare const TRANSCRIBE_STREAM_MS = 45000; /** The note a deadline puts on the bubble. A distinct wording, because "Whisper failed" reads as * the user's key or their audio being wrong, and neither is the case here. */ export declare const TRANSCRIBE_TIMEOUT_MESSAGE = "Transcription timed out \u2014 the audio never came back."; interface SpeechRecognitionAlternative { readonly transcript: string; } interface SpeechRecognitionResultLike { readonly isFinal: boolean; readonly length: number; readonly [index: number]: SpeechRecognitionAlternative; } interface SpeechRecognitionEventLike { readonly resultIndex: number; readonly results: { readonly length: number; readonly [index: number]: SpeechRecognitionResultLike; }; } interface SpeechRecognitionErrorEventLike { readonly error: string; } interface SpeechRecognitionLike { continuous: boolean; interimResults: boolean; lang: string; onresult: ((e: SpeechRecognitionEventLike) => void) | null; onerror: ((e: SpeechRecognitionErrorEventLike) => void) | null; onend: (() => void) | null; /** Fired when the UA BEGINS CAPTURING AUDIO. The one honest proof that this recognizer got the * device, and therefore the only event that may end a failure run (#425). Optional because a * stub may not have it — where it is absent a transcript is the fallback proof. */ onaudiostart?: (() => void) | null; start(): void; stop(): void; /** Standard on the real API; optional here because a stub may not have it. `stop()` asks for * a final result, `abort()` ends capture immediately — the fallback when `stop()` throws. */ abort?(): void; } type SpeechRecognitionCtor = new () => SpeechRecognitionLike; declare global { interface Window { SpeechRecognition?: SpeechRecognitionCtor; webkitSpeechRecognition?: SpeechRecognitionCtor; } } export interface SttOptions { apiKey?: string; language?: string; /** Transcription model for the OpenAI provider. Defaults to {@link DEFAULT_STT_MODEL}. */ model?: string; /** Vocabulary-bias prompt (see ./prompt.ts) so domain words aren't mis-heard. */ transcribePrompt?: string; onResult?: (text: string, isFinal: boolean) => void; /** `detail` accompanies a browser-recognizer failure only (#425): how many times in a row this * recognizer has failed, and how long the burst has lasted. An identical message is dropped * client-side within 30s and collapsed onto one durable row per hour, so without the counter a * single row cannot tell a turn-boundary transient from a loop that has been re-activating the * mic for an hour. */ onError?: (error: string, detail?: MicErrorDetail) => void; onEnd?: () => void; /** Fired with the raw recorded audio for a transcribed turn (Whisper only) so the * caller can save it for replay. Browser dictation has no blob → never fires. */ onAudio?: (blob: Blob) => void; /** * The live browser-dictation gate's reading, read AT THE MOMENT the clip is judged. * * A function rather than a value because the flags change during the recording, and the question * ("did real words happen this turn?") is only meaningful at the decision. The same snapshot * `use-voice` hands to `planTurnClose` and `planNoiseRejection`, so all three answer it * identically — one shape, and since #535 one precedence rule (`speechVerdict`). */ gate?: () => GateEvidence; /** The speech gate's verdict on a finished clip, with the numbers behind it — so a discard * before upload can leave a durable row instead of nothing at all (#510 criterion 2). */ onClipGate?: (verdict: ClipVerdict, snapshot: LevelSnapshot) => void; } export declare class VoiceStt { provider: string; apiKey: string; language: string; model: string; transcribePrompt: string; onResult: (text: string, isFinal: boolean) => void; onError: (error: string, detail?: MicErrorDetail) => void; onEnd: () => void; onAudio: (blob: Blob) => void; gate: (() => GateEvidence) | undefined; onClipGate: (verdict: ClipVerdict, snapshot: LevelSnapshot) => void; listening: boolean; private _rec; /** The failure run over the device (#425). On the INSTANCE rather than in `_startBrowser`'s * closure because the run belongs to the device, not to a recognizer object — Chrome's * `start()` throws after an abort, and the recovery for that is to build a fresh one, which * would otherwise reset the backoff to zero every time. For the same reason it is now cleared * only by PROOF OF CAPTURE and not by a benign code: `aborted` is what this class's own * `stop()` raises, and `stop()` happens at every turn boundary — which is where the contention * is. See {@link MicFailureRun}. */ private _micRun; private _micRetryTimer; /** This recognizer's identity in the shared device handoff (#425). Per INSTANCE: the control * listener and the main dictation recognizer are two `VoiceStt`s over one microphone. */ private _owner; /** Is a browser recognition session open (started, no `onend` since)? Only then does `stop()` * leave a close for the next consumer to wait on. */ private _sessionOpen; private _mediaRec; private _stream; /** Set by stopDiscard(): the pending recording is dropped instead of transcribed. */ private _discard; /** When the current recording started — used to drop sub-threshold captures. */ private _recStartedAt; /** Loudest mic level seen during the current recording, fed by noteLevel(). */ private _peakLevel; /** * How many levels `noteLevel` was given for the current recording. * * The distinction `_peakLevel` alone cannot make, and the one #490 got wrong: **0 frames is no * measurement, not a measurement of silence.** `noteLevel`'s only caller is a * `requestAnimationFrame` tick, and rAF does not run for a hidden document — so a backgrounded * console produces 0 frames for a clip full of speech, and reading that as "no speech" discards * every clip the user records while looking at something else. See `planClipGate`. */ private _levelFrames; /** The last clip handed to transcription, kept so a failed attempt can be RETRIED with the same * audio (#421) instead of asking the user to say it again. Cleared on a fresh recording. */ private _lastBlob; /** * Per-turn adaptive ambient noise floor from the VAD (`VadState.noiseFloor`). * Set by the audio monitor loop once the noise-sampling window closes * (`vadStateRef.current.noiseFloor`). `-1` = not yet available, fall back to the fixed * `VOICE_FLOOR`. Used by the `onstop` speech gate to raise the `hadSpeech` threshold when * the room is noisy, so steady ambient sound never registers as a real clip. */ _noiseFloor: number; /** The recorder's mic stream (Whisper mode) so the audio meter can reuse it * instead of opening a SECOND getUserMedia — a second capture mutes the recorder * on iOS Safari, yielding silent audio and empty transcriptions. */ get stream(): MediaStream | null; constructor(provider: string, opts?: SttOptions); start(): Promise; stop(): void; /** Stop every mic track and drop the stream. Safe to call twice. */ private _releaseStream; /** Stop the Whisper recorder but DROP the audio (no transcription). Used to * recycle a silent recording when the mic has sat open with no speech — avoids * uploading a long, mostly-silent blob to Whisper. Still fires onEnd so * conversation mode reopens the mic. No-op for browser dictation. */ /** * Feed the current mic level (0–1) from the caller's analyser loop. * * Deliberately not gated on listening mode: the auto-VAD runs only in hands-free, so gating * this the same way would leave tap-to-talk — the mode where a stray double-tap records pure * silence — with no speech gate at all. */ noteLevel(level: number): void; /** * What the analyser measured for the clip in hand — the numbers a drop has to be diagnosed * with (#510 criterion 5). Before this the durable log carried only `{code}` / `{transcript}`, * so "it didn't hear me" was answerable only by reading the source and guessing at the inputs. */ levelSnapshot(): LevelSnapshot; stopDiscard(): void; private _startBrowser; /** * Is there a clip a failed transcription could be retried with (#421)? The blob is still in * hand after any failure, so Retry is a re-POST rather than "say that again" — which matters * most for the failure this was written for, a timeout, where the user has already waited. */ canRetryTranscription(): boolean; /** Re-send the last clip. The caller owns the UI state; this only redoes the request. */ retryTranscription(): Promise; private _startRecording; private _transcribeWhisper; /** * Read the streaming-transcription SSE: fire interim `onResult(partial, false)` on * each delta (live words in the input) and a single `onResult(final, true)` at the * end. If the stream breaks after some text, we still emit what we have rather than * losing the turn. */ private _readTranscriptionStream; } export {}; //# sourceMappingURL=stt.d.ts.map