/** * Provider-agnostic speech-to-text domain types for daemon transcription. * * These types define the boundary between callers that need audio transcription * and the concrete STT provider implementations. The goal is to let daemon * callsites program against a single typed interface so that provider swaps are * localized to the adapter layer. * * Two execution modes are supported: * - **Batch** — a single audio buffer is sent and a final transcript returned. * - **Streaming** — audio chunks are sent over a persistent session, and the * provider emits partial/final transcript events in real time. */ // --------------------------------------------------------------------------- // Provider identity // --------------------------------------------------------------------------- /** * Canonical provider identifiers for daemon-hosted STT backends. * Extend this union as new providers are integrated. */ export type SttProviderId = | "openai-whisper" | "deepgram" /** * Deepgram's Flux conversational speech model. Shares the `deepgram` * credential and is streaming-only: Flux has no batch endpoint. */ | "deepgram-flux" | "google-gemini" | "xai" | "vellum"; /** * Telephony-specific STT capability class. * * Describes the provider's native audio-ingestion capability when used in * a telephony context. This is a **capability classification** consumed by * `resolveTelephonySttCapability` (providers/speech-to-text/resolve.ts) — * every supported provider transcribes call audio server-side on the * media-stream transport; this field describes how it ingests that audio. * * - `"realtime-ws"` — provider offers a WebSocket streaming endpoint suitable * for low-latency telephony audio (e.g. Deepgram live transcription). * - `"batch-only"` — provider supports only REST batch transcription as its * native capability; call audio is transcribed in incremental batches. * - `"none"` — provider has no telephony support. */ export type TelephonySttMode = "realtime-ws" | "batch-only" | "none"; /** * Conversation streaming STT support mode. * * Describes how a provider can participate in real-time conversation * streaming when used for chat message capture (chat composer and iOS * input bar). * * - `"realtime-ws"` — provider offers a native WebSocket streaming endpoint * that accepts audio chunks and emits partial/final transcript events * with low latency (e.g. Deepgram live transcription). * - `"incremental-batch"` — provider does not offer true streaming but can * be polled with incremental audio batches to approximate streaming * behaviour (e.g. Google Gemini multimodal). * - `"none"` — provider has no conversation streaming support; callers * should fall back to batch transcription. */ export type ConversationStreamingMode = | "realtime-ws" | "incremental-batch" | "none"; /** * Turn-detection support mode. * * Describes whether a provider decides end-of-turn itself, in-band with the * audio it is already transcribing, rather than leaving the boundary to the * session's local silence timer and front-door hold verdict. * * - `"provider"`: the provider emits turn lifecycle events (`turn-start`, * `turn-end`) on its transcript stream, and a live-voice session may let * those events commit the turn. * - `"none"`: the provider transcribes only; the local silence boundary owns * end-of-turn. * * A provider declaring `"provider"` should number its turns: the staleness * check prefers the turn index carried on the event, and degrades to the * session's local VAD generation counter when one is absent, which understates * the recorded latency for a turn the caller resumed and stopped again. */ export type SttTurnDetectionMode = "provider" | "none"; // --------------------------------------------------------------------------- // Boundary identifier // --------------------------------------------------------------------------- /** * Runtime boundary through which STT is executed. * - `daemon-batch` — transcription runs in the daemon process via a REST API * call to the provider (e.g. OpenAI Whisper). * - `daemon-streaming` — transcription runs in the daemon process over a * persistent streaming session (e.g. WebSocket or incremental-batch loop). */ export type SttBoundaryId = "daemon-batch" | "daemon-streaming"; // --------------------------------------------------------------------------- // Call-context hints // --------------------------------------------------------------------------- /** * Optional metadata hints that a caller can supply when the transcription * originates from a phone-call context. These are advisory — providers may * ignore hints they do not support. * * This type is intentionally separate from the batch request so that * call-context awareness can be added incrementally without changing the * shape for non-call callers. */ export interface SttCallContextHints { /** BCP-47 language code for the expected speech (e.g. "en-US"). */ language?: string; /** Static vocabulary hints (proper nouns, domain terms) the ASR should prioritize. */ vocabularyHints?: string[]; /** Short natural-language prompt to bias the transcription model. */ prompt?: string; } // --------------------------------------------------------------------------- // Request / result // --------------------------------------------------------------------------- /** Input to a batch transcription call. */ export interface SttTranscribeRequest { /** Raw audio data (WAV, OGG, MP3, etc.). */ audio: Buffer; /** MIME type of the audio data (e.g. "audio/ogg", "audio/wav"). */ mimeType: string; /** Optional abort signal for cancellation / timeout. */ signal?: AbortSignal; /** * Optional call-context hints. Present when the transcription request * originates from a telephony call. Providers may use these to improve * recognition accuracy. */ callContext?: SttCallContextHints; } /** Successful transcription output. */ export interface SttTranscribeResult { /** The transcribed text, trimmed. Empty string for silence. */ text: string; } // --------------------------------------------------------------------------- // Normalized error categories // --------------------------------------------------------------------------- /** * Normalized error categories that callers can branch on without coupling to * provider-specific error shapes or HTTP status codes. */ export type SttErrorCategory = /** The provider rejected the request due to invalid or missing credentials. */ | "auth" /** The provider rate-limited the request. */ | "rate-limit" /** The request or response timed out. */ | "timeout" /** The audio payload was rejected (unsupported format, too large, etc.). */ | "invalid-audio" /** Any other provider-side or network failure. */ | "provider-error"; /** A transcription error enriched with a normalized category. */ export class SttError extends Error { readonly category: SttErrorCategory; /** * When true, {@link message} is already user-facing, provider-appropriate * copy and must be surfaced verbatim rather than rewritten by * `describeSttFailure`. Set by adapters (e.g. managed speech) whose failures * carry specific remediation — a credit top-up or platform reconnect — that * the generic BYOK "check your API key" copy would erase. */ readonly userFacing: boolean; constructor( category: SttErrorCategory, message: string, options?: { userFacing?: boolean }, ) { super(message); this.name = "SttError"; this.category = category; this.userFacing = options?.userFacing ?? false; } } // --------------------------------------------------------------------------- // Batch transcriber interface // --------------------------------------------------------------------------- /** * Daemon-hosted batch transcriber contract. * * Implementations accept a buffer of audio data and return a transcription * result. Errors propagate as raw provider errors (not wrapped in * {@link SttError}) so that callers relying on specific error identities * (e.g. `AbortError` for cancellation) continue to work. Callers that need * normalized error categories should wrap calls with `normalizeSttError()` * from `daemon-batch-transcriber.ts`. */ export interface BatchTranscriber { /** Which provider backs this transcriber. */ readonly providerId: SttProviderId; /** Which runtime boundary this transcriber operates in. */ readonly boundaryId: SttBoundaryId; /** * Transcribe a chunk of audio. * * Rejects with the raw provider error on failure. Use * `normalizeSttError()` to convert to an {@link SttError} with a * structured {@link SttErrorCategory}. */ transcribe(request: SttTranscribeRequest): Promise; } // --------------------------------------------------------------------------- // Streaming client events (client -> daemon) // --------------------------------------------------------------------------- /** * Events that a client sends to the daemon streaming session. * * The discriminated `type` field lets the runtime session handler * dispatch to the correct streaming adapter method without coupling * to transport-level framing (WebSocket opcodes, HTTP/2 frames, etc.). */ export type SttStreamClientEvent = | SttStreamClientAudioEvent | SttStreamClientStopEvent; /** A chunk of audio data to be transcribed. */ export interface SttStreamClientAudioEvent { readonly type: "audio"; /** Raw audio data for the current chunk. */ readonly audio: Buffer; /** MIME type of the audio data (e.g. "audio/webm", "audio/pcm"). */ readonly mimeType: string; } /** Signals that the client has finished sending audio. */ export interface SttStreamClientStopEvent { readonly type: "stop"; } // --------------------------------------------------------------------------- // Streaming server events (daemon -> client) // --------------------------------------------------------------------------- /** * Events that the daemon streaming session emits to the client. * * The discriminated `type` field allows clients to handle partial * and final transcripts, errors, and session lifecycle signals in a * type-safe manner. */ export type SttStreamServerEvent = | SttStreamServerPartialEvent | SttStreamServerFinalEvent | SttStreamServerFinalizedEvent | SttStreamServerTurnStartEvent | SttStreamServerEagerTurnEndEvent | SttStreamServerTurnResumedEvent | SttStreamServerTurnEndEvent | SttStreamServerErrorEvent | SttStreamServerClosedEvent; /** * A partial (interim) transcript — may be revised by subsequent * partial or final events. */ export interface SttStreamServerPartialEvent { readonly type: "partial"; /** Interim transcript text. May change with subsequent events. */ readonly text: string; // Provider-emitted speaker label; undefined when diarization is disabled or unsupported. Consumers cross-check with channel-specific signals (e.g., Meet DOM). readonly speakerLabel?: string; /** * Provider-emitted confidence score in [0, 1]. Undefined when the * provider does not surface confidence on interim chunks. */ readonly confidence?: number; /** * All detected languages of this chunk in dominance order (first entry * dominant), already normalized to lowercase base subtags (e.g. "en", * "hi") by the emitting adapter; consumers must not re-normalize. Only * providers with code-switching metadata (currently Deepgram nova-3 * `multi`) populate this. Absence means "unknown", never English. */ readonly languages?: readonly string[]; } /** * A final (committed) transcript — this segment will not be revised. */ export interface SttStreamServerFinalEvent { readonly type: "final"; /** Committed transcript text for a completed speech segment. */ readonly text: string; // Provider-emitted speaker label; undefined when diarization is disabled or unsupported. Consumers cross-check with channel-specific signals (e.g., Meet DOM). readonly speakerLabel?: string; /** * Provider-emitted confidence score in [0, 1]. Undefined when the * provider does not surface confidence on this chunk. */ readonly confidence?: number; /** * All detected languages of this chunk in dominance order (first entry * dominant), already normalized to lowercase base subtags (e.g. "en", * "hi") by the emitting adapter; consumers must not re-normalize. Only * providers with code-switching metadata (currently Deepgram nova-3 * `multi`) populate this. Absence means "unknown", never English. */ readonly languages?: readonly string[]; /** * True when this final is the flush response to * {@link StreamingTranscriber.finalizeUtterance} — i.e. it commits audio * that was buffered before the finalize request, not new speech. * Undefined on ordinary finals and on providers without finalize. */ readonly fromFinalize?: boolean; } /** * All provider-buffered audio has been flushed into `final` event(s) in * response to {@link StreamingTranscriber.finalizeUtterance}. Emitted * exactly once per finalize request, after the flushed `final` event(s). * The stream stays open and continues to accept audio. */ export interface SttStreamServerFinalizedEvent { readonly type: "finalized"; } // --------------------------------------------------------------------------- // Turn-detection events // // Consumers that ignore these keep committing transcripts from `final` // exactly as they do without a turn-detecting provider. // --------------------------------------------------------------------------- /** * The provider's turn model detected the start of a user turn. * * Emitted only by providers with model-integrated turn detection; providers * without it never emit these. */ export interface SttStreamServerTurnStartEvent { readonly type: "turn-start"; /** * Provider's index for the turn being opened, monotonically increasing * within a stream. Undefined when the provider does not number its turns. */ readonly turnIndex?: number; } /** * The provider's turn model speculates the user has finished, ahead of its * committed end-of-turn decision. A {@link SttStreamServerTurnResumedEvent} * retracts the speculation; a {@link SttStreamServerTurnEndEvent} confirms it. * * Emitted only by providers with model-integrated turn detection; providers * without it never emit these. */ export interface SttStreamServerEagerTurnEndEvent { readonly type: "eager-turn-end"; /** Transcript text as of the speculated turn end. */ readonly text: string; } /** * The user kept speaking after an {@link SttStreamServerEagerTurnEndEvent}, * retracting the speculated turn end. * * Emitted only by providers with model-integrated turn detection; providers * without it never emit these. */ export interface SttStreamServerTurnResumedEvent { readonly type: "turn-resumed"; } /** * The provider's turn model committed an end of turn: the user has finished * speaking and a reply can be dispatched. * * Emitted only by providers with model-integrated turn detection; providers * without it never emit these. */ export interface SttStreamServerTurnEndEvent { readonly type: "turn-end"; /** Committed transcript text for the completed turn. */ readonly text: string; /** * Provider-emitted end-of-turn confidence in [0, 1]. Undefined when the * provider does not surface a score. */ readonly confidence?: number; /** * Provider's index for the turn being closed, matching the * {@link SttStreamServerTurnStartEvent} that opened it. A consumer can * compare it against the newest turn it has seen opened to tell an * end-of-turn for the turn still in progress from one the provider has * already superseded. Undefined when the provider does not number its * turns. */ readonly turnIndex?: number; } /** An error occurred during streaming transcription. */ export interface SttStreamServerErrorEvent { readonly type: "error"; /** Normalized error category for caller branching. */ readonly category: SttErrorCategory; /** Human-readable error description. */ readonly message: string; } /** The streaming session has closed (no more events will be emitted). */ export interface SttStreamServerClosedEvent { readonly type: "closed"; } // --------------------------------------------------------------------------- // Streaming transcriber interface // --------------------------------------------------------------------------- /** * Daemon-hosted streaming transcriber contract. * * Implementations manage a persistent session that accepts audio chunks * and emits partial/final transcript events. The runtime session * orchestrator (PR 5) drives this interface from the gateway WebSocket * path. * * Lifecycle: * 1. Call {@link start} to open the provider session. * 2. Feed audio chunks via {@link sendAudio}. * 3. Optionally call {@link finalizeUtterance} between utterances to * flush buffered audio into finals while keeping the stream open. * 4. Call {@link stop} when the client finishes recording. * 5. The `onEvent` callback receives server events until `closed`. */ export interface StreamingTranscriber { /** Which provider backs this transcriber. */ readonly providerId: SttProviderId; /** Which runtime boundary this transcriber operates in. */ readonly boundaryId: "daemon-streaming"; /** * Open the streaming session with the provider. * * Must be called once before {@link sendAudio}. Rejects if the * provider session cannot be established. */ start(onEvent: (event: SttStreamServerEvent) => void): Promise; /** * Feed a chunk of audio into the streaming session. * * Callers must not call this before {@link start} resolves or after * {@link stop} has been called. */ sendAudio(audio: Buffer, mimeType: string): void; /** * Flush all buffered audio into final transcript(s) without closing * the stream. * * Emits `final` event(s) for pending audio followed by one `finalized` * event; the stream stays open for more audio. Optional — callers must * feature-detect this method and fall back to {@link stop} (full * teardown) when the provider does not support it. */ finalizeUtterance?(): void; /** * Signal that the client has finished sending audio. * * The provider may emit additional final events after stop is called. * The session is fully closed when the `onEvent` callback receives a * `closed` event. */ stop(): void; }