import { type PipelineBus } from "./pipeline-bus.js"; import type { VoicePlugin, PluginConfig } from "./plugin-contract.js"; import { type IdleTimeoutConfig } from "./idle-timeout.js"; import { type ConversationEvent } from "./conversation-event.js"; import type { UsageStage } from "./packets.js"; import { SessionState } from "./packets.js"; import { type InteractionPolicy } from "./interaction-policy.js"; import { type LoudnessConfig } from "./audio/loudness.js"; import { type MetricsExporter } from "./observability.js"; import { type ObservabilityLayer } from "./observability.js"; import { type Scheduler } from "./scheduler.js"; export interface VoiceAgentSessionConfig { /** Plugin configurations, keyed by plugin name. */ plugins: Record; /** Idle timeout configuration. */ idleTimeout?: Partial; /** PipelineBus configuration. */ busConfig?: { mainCapacity?: number; bgCapacity?: number; criticalBatchSize?: number; /** Diagnostic: how long each packet waited between push and dispatch. */ onQueueDelay?: (kind: string, delayMs: number) => void; }; /** * Maximum ms to wait for an STT final transcript after audio injection stops. * When this timer fires, asks the streaming STT provider to flush buffered audio. * Default: 7000 (endpointing + 2s grace) */ sttForceFinalizeTimeoutMs?: number; /** * Minimum characters before the FIRST spoken chunk of a turn may be dispatched at * a clause boundary rather than waiting for a full sentence. Only the first * dispatch gates time-to-first-audio. 0 disables (always wait for a sentence). * * Default 25. An earlier default of 45 was measured against real replies and * never fired: it starts the scan PAST the clause boundary in a typical * sentence ("The deadline is March first," is 28 chars). Set 0 for HTTP-per-request TTS, where * an extra chunk means an extra synthesis call and can hurt continuity. */ firstFragmentMinChars?: number; /** * Minimum sustained user-speech duration (ms) during assistant playback before a * barge-in is committed. Filters transient noise, clicks, and very short blips that * would otherwise falsely cut off the agent. The agent keeps speaking until the * user's speech is sustained past this threshold, then interruption fires * immediately. Set to 0 to disable the gate and interrupt on the first VAD speech * frame (legacy behavior). Default: 280 ms. */ minInterruptionMs?: number; /** * When true (default), barge-in requires sustained speech from the enrolled * primary speaker (first user turn fingerprint) in addition to G1's time gate. * Non-primary / echo speech emits `interrupt.suppressed_non_primary`. When no * profile is enrolled yet, falls back to G1-only behavior. */ primarySpeakerBargeInEnabled?: boolean; /** * When true, emit a short discourse connective via TTS at endpoint (before LLM * TTFB) and splice the real response in when it arrives. Off by default. */ latencyFillerEnabled?: boolean; /** * Maximum ms after a user turn ends to wait for first assistant audio before * emitting a vaqi.missed_response metric (VAQI-M). 0 disables the check. * Default: 4000. */ vaqiMissedResponseMs?: number; /** * Max ms of silence from the TTS provider AFTER it has begun producing audio for a * turn before the output is treated as a stalled provider. Guards against a TTS * provider that goes silent mid-utterance without `tts.end` or an error (dead air). * Armed only after the first `tts.audio`, so first-audio latency (which can be many * seconds on some providers, e.g. Gemini) is never watchdogged. On breach, a * recoverable `tts.error` (NetworkTimeout) is emitted so the turn fails visibly * instead of hanging. 0 disables. Default: 15000. */ ttsStallMs?: number; /** Optional per-session Int16 RMS normalization for outbound assistant audio. Disabled by default. */ outboundLoudness?: LoudnessConfig; /** * Max ms of silence on inbound user audio while the session is Ready before a * recoverable transport warning is emitted. Continuous streams (telephony, open * mic) should set this; push-to-talk / headless sessions leave at 0 (disabled). * Default: 0. */ inputCadenceTimeoutMs?: number; /** * Spoken fallback when the reasoning (LLM) layer fails a turn with a recoverable * error. "Never fail silently" (Deepgram guide Ch3): rather than ending the turn in * unexplained silence, the agent speaks this line via the normal TTS path (which is * unaffected by an LLM failure). Empty string disables. Default: a brief apology. * (TTS/STT-failure fallback needs canned audio / a clarification prompt — out of scope.) */ errorFallbackText?: string; /** * G3 (RFC bimodel-delegate-seam): ms a tool call may stay pending before the * `tool_call_cue` session event fires its time-triggered `"delayed"` phase — the * "still working" cue clients render during a long reasoner wait (cf. Vapi's * `request-response-delayed` + `timingMilliseconds`). 0 disables the delayed * phase; started/complete/failed always fire. Default: 2000. */ delayCueAfterMs?: number; /** * Which component owns turn boundary (EOS) for this session. Defaults to * provider STT ownership; Smart Turn sessions must opt in explicitly. */ endpointingOwner?: "provider_stt" | "smart_turn" | "timer"; /** The front model owns full-duplex interaction (turn-taking + barge-in). When true, the session's * InteractionPolicy runs observe-only (DeferInteractionPolicy) — Syrinx does not drive its own * turn/interrupt decisions; the front's native decisions stand. Default: false (Syrinx drives). * A realtime factory sets this from RealtimeAdapter.caps.supportsFullDuplex. */ fullDuplex?: boolean; /** When true, Syrinx suppresses policy-timed backchannel cue packets (front/provider owns them). */ emitsBackchannel?: boolean; /** * Optional interaction policy injected by the caller (learned controllers, Smart Turn policy, etc.). * When omitted, the session uses RuleBasedInteractionPolicy. When `fullDuplex` is true, the * coordinator runs observe-only via DeferInteractionPolicy regardless of this setting. */ interactionPolicy?: InteractionPolicy; /** Config passed to `interactionPolicy.initialize` when the injected policy is lifecycle-capable. */ interactionPolicyConfig?: Record; readonly metricsExporter?: MetricsExporter; readonly scheduler?: Scheduler; readonly observability?: { readonly sessionId?: string; readonly provider?: string; readonly model?: string; readonly region?: string; readonly layer?: ObservabilityLayer; }; } export interface VoiceAgentSessionEvents { user_started_speaking: (event: { tsMs: number; turnId: string; }) => void; user_stopped_speaking: (event: { tsMs: number; turnId: string; }) => void; user_input_partial: (event: { tsMs: number; turnId: string; text: string; }) => void; user_input_final: (event: { tsMs: number; turnId: string; text: string; confidence: number; }) => void; agent_text_delta: (event: { tsMs: number; turnId: string; delta: string; }) => void; agent_tool_call: (event: { tsMs: number; turnId: string; id: string; name: string; args: Record; }) => void; agent_tool_result: (event: { tsMs: number; turnId: string; id: string; result: string; durationMs: number; }) => void; delegate_query: (event: { tsMs: number; turnId: string; query: string; toolId?: string; toolName?: string; }) => void; delegate_result: (event: { tsMs: number; turnId: string; query: string; answer: string; durationMs: number; grounded: boolean; toolId?: string; toolName?: string; control?: { name: string; payload: unknown; }; blocked?: { userFacingMessage: string; payload?: unknown; }; }) => void; /** * G3: typed preamble/filler lifecycle for a pending tool call (Vapi-shaped: * started / delayed / complete / failed). `delayed` is time-triggered by * `delayCueAfterMs` while the call is still pending; `failed` fires on an LLM/bridge * error, a barge-in, or a superseding turn while pending (R5). Transports surface * these as `tool_call_*` wire messages — the standard "thinking" cue. */ tool_call_cue: (event: { tsMs: number; turnId: string; phase: "started" | "delayed" | "complete" | "failed"; toolId: string; toolName: string; afterMs?: number; }) => void; /** * Per-turn latency decomposition, timestamped at the turn's first TTS audio. * When generation is still active at first audio, emission waits for `tts.end` so * provider passes that follow a spoken tool preamble are included in the totals. * `ttfaMs` is anchored to the real end of user speech (VAD speech-end, falling * back to the endpoint decision) — `fillerUsed` flags turns where a latency filler spoke * first, and `backchannelUsed` flags turns where a wait-gap cue played before the answer. * Decomposition: eouDelayMs (speech end → endpoint) + llmTtftMs (endpoint → * first LLM delta) + textAggregationMs (first LLM delta → first TTS text) + * ttsTtfbMs (first TTS text dispatched → first audio). A stage is omitted when the * front does not produce that Syrinx packet for this turn: realtime fronts may emit * provider audio directly, and provider-owned endpointing may complete after audio. * `unattributedMs` is the explicit residual after subtracting only the stages present. */ turn_latency: (event: { tsMs: number; turnId: string; ttfaMs: number; anchor: "speech_end" | "eos"; eouDelayMs?: number; llmTtftMs?: number; textAggregationMs?: number; ttsTtfbMs?: number; /** * Time this turn's latency-critical packets spent waiting to be dispatched. * Every other stage is derived from packet timestamps stamped at CREATION, so * none of them can see a handler parking the drain loop. Non-zero here means * some of the stages above are reporting less than the caller experienced. */ queuedMs?: number; unattributedMs: number; llmCallCount?: number; llmPassTtftMs?: readonly number[]; fillerUsed: boolean; backchannelUsed: boolean; }) => void; agent_first_audio: (event: { tsMs: number; turnId: string; }) => void; agent_finished: (event: { tsMs: number; turnId: string; } & Record) => void; /** * End-of-session usage manifest — the total billable resource this session consumed, * summed per stage. Emitted once at close. This is the metering seam a host reads to * bill, cap spend, or attribute cost to a tenant; today only the LLM stage is populated * (STT/TTS producers are not yet wired), so `stages` may hold a single entry. */ usage: (event: { tsMs: number; stages: readonly SessionStageUsage[]; }) => void; error: (event: { tsMs: number; stage: string; category: string; message: string; }) => void; closed: (event: { tsMs: number; reason: string; }) => void; state_changed: (event: { tsMs: number; from: SessionState; to: SessionState; }) => void; } /** Per-stage usage totals for a session. Absent fields were never reported by that stage. */ export interface SessionStageUsage { readonly stage: UsageStage; readonly inputTokens?: number; readonly outputTokens?: number; readonly totalTokens?: number; readonly cachedInputTokens?: number; readonly reasoningTokens?: number; readonly audioSeconds?: number; readonly characters?: number; } export declare class VoiceAgentSession { readonly bus: PipelineBus; readonly debugEvents: ReadableStream; private readonly config; private readonly sttForceFinalizeTimeoutMs; private _state; private plugins; private initSteps; private idleTimeout; private modeSwitcher; private debugPush; private eventListeners; private currentTurnId; private busStartPromise; private closePromise; private readonly scheduler; private readonly ttsPlayout; private interruptedGenerationContextIds; private generatingContextIds; private ttsTextBuffers; /** Dispatch lag accumulated by this turn's latency-critical packets. See emitTurnLatency. */ private turnQueuedMs; /** The turn currently accruing queue delay. Packets do not carry it themselves. */ private currentLatencyContextId; private readonly minInterruptionMs; private readonly primarySpeakerGate; private readonly ruleBasedPolicy; private readonly injectedInteractionPolicy; private readonly activeInteractionPolicy; private readonly interaction; private readonly latencyFiller; private firstLlmDeltaReceived; private readonly vaqiMissedResponseMs; private readonly ttsStallMs; private readonly outboundLoudnessConfig; private readonly outboundLoudnessState; private readonly inputCadenceTimeoutMs; private readonly watchdogs; private readonly observabilityObserver; private readonly metricsExporter; private readonly observabilityDims; private turnUserStoppedAtMs; private turnTimings; private pendingTurnLatency; /** Running usage totals per stage, summed across the session; emitted at close. */ private readonly usageByStage; private speakerEnrollmentContextId; private firstTtsAudioFired; private readonly pendingInteractionPlayoutTimers; private readonly errorFallbackText; private fallbackInjectedContexts; private readonly delayCueAfterMs; private pendingToolCues; private readonly endpointingOwner; private readonly firstFragmentMinChars; private readonly fullDuplex; private readonly emitsBackchannel; private userSpeaking; private lastFinalizedContextId; private readonly sttPartialWordTimings; private readonly turnLocalizationStates; private readonly emittedTurnLocalizations; constructor(config: VoiceAgentSessionConfig); private get turnArbiter(); get state(): SessionState; get currentContextId(): string; /** Register a plugin. Must be called before start(). */ registerPlugin(name: string, plugin: VoicePlugin): void; /** Start the session. Runs init chain, starts bus draining. */ start(): Promise; /** * Best-effort warm of registered plugins' remote/expensive resources. Call after start() to * wake scaled-to-zero endpoints before the first user turn. The host decides when to invoke * this — it is not called automatically from start(). Never throws; failed plugin prewarms * are swallowed and may emit a prewarm.failed metric on the bus. */ prewarm(): Promise; /** Shut down the session. Runs finalize chain in reverse order. */ close(): Promise; private closeOnce; /** Switch between text and audio mode. */ switchMode(mode: "text" | "audio"): Promise; requestClientInterrupt(contextId: string): void; on(event: K, handler: VoiceAgentSessionEvents[K]): void; off(event: K, handler: VoiceAgentSessionEvents[K]): void; private emit; private wireBusHandlers; private handleUserAudio; private observeAudioFrame; private handleSttAudio; private handleUserText; private handleSttPartial; private handleSttInterim; private handleSttResult; private handleVadAudioForSpeakerGate; private shouldEnrollPrimarySpeaker; private observeSttForBargeIn; private handleVadSpeechStarted; private handleVadSpeechActivity; private handleVadSpeechEnded; /** * A realtime front rotates its contextId when the provider starts responding * (`RealtimeBridge.onResponseStarted`), so the user-side anchors (`speechEndedMs`, `eosMs`) * are recorded under the PREVIOUS context while `tts.audio` — and therefore the * `turn_latency` emit — lands under the new one. Without carrying the record across the * rotation, `emitTurnLatency` finds no anchor and silently drops every native turn. * * Only fills gaps: anything the new context already recorded wins. */ private carryTurnTimingAcrossContextChange; private timingFor; /** Emit a deferred turn_latency if one is pending, then clear it. Safe to call twice. */ private flushPendingTurnLatency; private emitTurnLatency; private handleTurnComplete; private handleEosInterim; private handleLlmDelta; private handleLlmDone; private bufferTtsText; private flushTtsText; private cancelLatencyFillerTurn; private handleLlmToolCall; private emitToolCallCue; /** G3: resolve one pending tool call with a terminal cue phase. */ private resolveToolCue; /** G3: fail every pending tool call for a context (error / barge-in / supersede). */ private failPendingToolCues; private handleDelegateQuery; private handleDelegateResult; private handleLlmToolResult; private handleConversationMetric; private emitAcousticSignal; private localizationStateFor; private markInfrastructureBreach; private markConversationFlag; private emitTurnLocalization; private scheduleTurnLocalization; private static readonly USAGE_FIELDS; private handleUsageRecorded; private emitSessionUsage; private handleTtsAudio; private normalizeOutboundAudio; private handleTtsEnd; private scheduleInteractionPlayoutTick; private cancelInteractionPlayoutTick; private handleTtsPlayoutProgress; private handleInterruptDetected; /** * Cancel a stale prior-turn generation/playout when a new turn supersedes it * (L1). Mirrors the interrupt teardown but without the barge-in metrics — this * is a turn boundary, not a user interruption. Stops leftover TTS audio, aborts * the LLM, and drops late deltas/audio for the stale context. */ private cancelStaleGeneration; private handleComponentError; private maybeSpeakErrorFallback; private handleInitFailed; private handleInjectMessage; private handleSttReconfigure; private handleDisconnect; private buildInitChain; private applyEndpointingOwnerInvariant; private shouldInitializePlugin; private emitDebug; private latestActiveTtsContextId; } //# sourceMappingURL=voice-agent-session.d.ts.map