import type { CostBudgetTracker, CostLimitContext } from './cost-budget.js'; import type { ModelUsage } from './model.js'; import type { ToolDef } from './tools.js'; /** * Bidirectional voice / audio streaming surface. fabric-harness ships an * OpenAI Realtime implementation; bring-your-own-vendor for Anthropic / * Gemini Live / on-prem TTS+ASR pipelines. * * Audio frames flow in raw bytes — the standard format is PCM 16-bit * little-endian at 24kHz mono (OpenAI Realtime default). Telephony * bridges (Twilio Media Streams μ-law 8kHz, etc.) resample at the edge. * * Tool execution is the *caller's* responsibility: a `tool_call` event * surfaces, the host code runs the tool through whatever governance gates * apply (approvals, cost caps, rate limits), then calls * `submitToolResult({ id, output })`. Voice sessions deliberately don't * own the loop machinery — this keeps the audio path lean and lets users * apply the v1-era policy primitives unchanged. */ export type VoiceAudioFormat = 'pcm16' | 'g711_ulaw' | 'g711_alaw'; export interface VoiceConnectOptions { /** Initial system instructions / persona. */ instructions?: string; /** Voice timbre id (provider-specific, e.g. 'alloy', 'nova', 'verse'). */ voice?: string; /** Audio format used on both directions. Default 'pcm16' at 24kHz mono. */ audioFormat?: VoiceAudioFormat; /** Tools the model can call. Same shape as session.prompt() tools. */ tools?: ToolDef[]; /** Turn-detection mode (provider-specific). Default 'server_vad'. */ turnDetection?: 'server_vad' | 'none'; /** Sample rate hint for the provider. Default 24000 (PCM16). */ sampleRate?: number; /** * Optional cost budget tracker. Observed on every `response_done` with * usage. Voice sessions thus participate in the same per-call / * per-session / per-tenant ceilings as text agents (v1.4 + v1.6). */ costBudget?: CostBudgetTracker; /** * Required when `costBudget.onExceed === 'approve'`. Returns true to * resume, false to throw `CostLimitExceededError`. */ requestCostLimitApproval?: (ctx: CostLimitContext) => Promise; /** Abort signal — closes the session when fired. */ signal?: AbortSignal; } /** * Events streamed from a `VoiceSession`. `audio_delta` carries raw audio * bytes; `text_delta` and `transcript` carry text; `tool_call` and * `response_done` mark structured boundaries; `error` is fatal. */ export type VoiceEvent = { type: 'session_open'; } | { type: 'audio_delta'; audio: Uint8Array; } | { type: 'text_delta'; delta: string; role: 'assistant' | 'user'; } | { type: 'transcript'; text: string; role: 'assistant' | 'user'; } | { type: 'tool_call'; id: string; name: string; input: unknown; } | { type: 'response_done'; usage?: ModelUsage; } | { type: 'cost_limit'; scope: 'call' | 'session' | 'cross-process'; observedUsd: number; limitUsd: number; scopeKey?: string; } | { type: 'error'; message: string; }; export interface VoiceToolResultInput { /** Tool-call id from the matching `tool_call` event. */ id: string; /** JSON-serializable output — stringified before sending. */ output: unknown; } export interface VoiceSession { /** Append a raw audio frame. Format matches `VoiceConnectOptions.audioFormat`. */ sendAudio(frame: Uint8Array): Promise; /** Send a text turn (the model responds with audio + text). */ sendText(text: string): Promise; /** Submit a tool-call result back to the model after a `tool_call` event. */ submitToolResult(input: VoiceToolResultInput): Promise; /** Cancel the in-progress response (barge-in). */ cancelResponse(): Promise; /** Async iterable of events. Closes when the session ends or aborts. */ events(): AsyncIterable; /** Tear down the underlying connection. */ close(): Promise; } export interface VoiceProvider { name: string; connect(options: VoiceConnectOptions): Promise; } //# sourceMappingURL=voice.d.ts.map