// SDK-native voice call — opens a live WebRTC voice session directly to the // provider using the credentials from startCall(), with NO dependency on Pouchy's // first-party connection wrappers (which are coupled to app stores / genui). This // is what lets a pure third-party embed do voice. // // - OpenAI Realtime: fully self-contained WebRTC (no deps). // - ElevenLabs Convai: dynamically imports '@elevenlabs/client' (an OPTIONAL peer // dependency) only when that provider is selected; if it's not installed the // call rejects with a clear message. // // Browser-only (needs WebRTC + getUserMedia). Returns a handle to hang up and to // inject live context mid-call (the bridge the server uses for voiceRelevant). import type { CallCredentials, CallPrivateStory, CallRecordingUnavailable, CallTrimmed, CallVideo, CallVideoUnavailable, CallWorldBrief } from './client'; import { CompanionError } from './errors'; import { VOICE_TOOL_VARIABLE, VOICE_TOOL_VARIABLE_PLAIN } from './voice-tool'; /** Lets the live voice call invoke the app's declared tools (pause_game, * set_feature, …). The companion's voice LLM calls a tool → invoke() runs the * app's onToolCall handler and resolves with a result string fed back to the * model. Wired by connectCall from the tools declared at createCompanion(). */ export interface VoiceToolBridge { tools: { name: string; description?: string; parameters?: Record }[]; invoke: (name: string, argsJson: string) => Promise; } /** Universal "host control" verbs the companion can call on ANY surface (web / * game / app / hardware), regardless of provider. They're generic on purpose — * the SPECIFIC intent rides in the string params, which YOUR surface interprets * — so a single fixed schema set serves every embed AND can be pre-registered on * the (otherwise static) ElevenLabs Convai agent. `invoke_action` is the * catch-all trigger; the SDK unwraps it to the named action so it reaches your * onToolCall just like a declared tool. The others pass through by name. */ export const HOST_CONTROL_TOOLS: VoiceToolBridge['tools'] = [ { name: 'invoke_action', description: 'Trigger a named action the current surface supports — e.g. pause, resume, start, stop, restart, next, previous, submit, confirm, cancel, refresh. Put the action name in "action" and any extra args in "params".', parameters: { type: 'object', properties: { action: { type: 'string', description: 'the action, e.g. "pause" / "resume" / "next"' }, params: { type: 'object', description: 'optional extra arguments for the action' } }, required: ['action'] } }, { name: 'set_feature', description: 'Turn a named feature/capability on or off (e.g. hints, captions, dark mode, a light, notifications, a mode).', parameters: { type: 'object', properties: { feature: { type: 'string' }, on: { type: 'boolean' } }, required: ['feature', 'on'] } }, { name: 'set_value', description: 'Set a named control to a value — volume, brightness, difficulty, speed, zoom, temperature, etc.', parameters: { type: 'object', properties: { name: { type: 'string' }, value: { description: 'the value: a number, string, or boolean' } }, required: ['name', 'value'] } }, { name: 'navigate', description: 'Go to a named view / screen / page / section / mode / level in the surface.', parameters: { type: 'object', properties: { to: { type: 'string' }, params: { type: 'object' } }, required: ['to'] } }, { name: 'highlight', description: "Draw the user's attention to a named element (guidance / onboarding / a tutorial step).", parameters: { type: 'object', properties: { target: { type: 'string' }, note: { type: 'string' } }, required: ['target'] } } ]; /** Names of the host-control verbs (for routing). */ export const HOST_CONTROL_TOOL_NAMES: readonly string[] = HOST_CONTROL_TOOLS.map((t) => t.name); /** Avatar gesture/expression tools the embodied ("形象") companion calls to emote * — drive a VRM/3D body's gestures and facial expression. These are offered on * EVERY embed (like the host verbs) so the companion can express itself without * each third party re-declaring them, and — crucially — so the SDK can register * a no-op handler for them on the ElevenLabs Convai path, where the shared agent * is pre-configured to call play_gesture / play_expression regardless of what the * embed declares. An integrator that actually renders an avatar opts in by * declaring (or handling) the same tool name and the SDK routes the call to them; * otherwise it answers with a silent no-op success (see invokeVoiceTool / emit), * so a pure audio or 2D embed can ignore avatars entirely and the companion's * emote calls never error. Enum values mirror the canonical GESTURE_CATEGORIES / * EXPRESSION_INTENTS the avatar runtime understands (kept inline so the SDK stays * self-contained). */ export const AVATAR_VISUAL_TOOLS: VoiceToolBridge['tools'] = [ { name: 'play_gesture', description: 'Play a full-body avatar gesture to express yourself physically (greeting, agreement, thinking, celebration, dance, …). No effect on surfaces without an avatar.', parameters: { type: 'object', properties: { category: { type: 'string', description: 'the kind of gesture', enum: [ 'greeting', 'agreement', 'disagreement', 'thinking', 'celebration', 'affection', 'shy', 'surprise', 'dance', 'point', 'pose' ] } }, required: ['category'] } }, { name: 'play_expression', description: "Set the avatar's facial expression to convey emotion (smile, laugh, wink, sad, surprised, …). No effect on surfaces without an avatar.", parameters: { type: 'object', properties: { intent: { type: 'string', description: 'the facial expression', enum: [ 'smile', 'laugh', 'wink', 'shy', 'pout', 'sad', 'angry', 'surprised', 'relaxed', 'smug', 'neutral' ] } }, required: ['intent'] } } ]; /** Remove Eleven v4 performed cues ("[laughs]", "[whispers]", "[short pause]") * from a call transcript line (0.64.0). A call opened with `audioTags: true` * lets the voice ACT OUT these cues, and the provider's transcript still * contains them; `connectCall` strips them before `onTranscript` and before * the transcript reaches memory. Exported for integrations that open the * ElevenLabs session themselves from `startCall` credentials. * * Only ASCII-word cues are removed (the cue vocabulary is English): a line's * own brackets — `【重点】`, `[1]` — are left alone. Padded (`[ laughs ]`), * full-width (`［laughs］`) and trailing-ellipsis (`[sighs…]`) spellings of a * cue count. */ export function stripAudioTags(text: string): string { return text .replace(audioTagPattern(), '') .replace(/[ \t]{2,}/g, ' ') .replace(/\s+([，,。.!！?？、])/g, '$1') .trim(); } /** The performed cues in a call transcript line, in order, without brackets * (0.70.0): `'[laughs] 好呀 [whispers] 秘密'` → `['laughs', 'whispers']`. The * same cues `stripAudioTags` removes, so the two always agree. `connectCall` * reports them on each companion line of a call where `audioTags` was * granted (`onTranscript`'s `audioTags`); exported for self-plumbed * ElevenLabs sessions. */ export function extractAudioTags(text: string): string[] { return [...text.matchAll(audioTagPattern())].map((m) => m[1]!.trim()); } /** The one cue shape both functions use: an ASCII cue word in ASCII `[ ]` or * full-width `［ ］` brackets, padding inside allowed, an optional trailing * `…` / `...` (CR-1140 — `[ laughs ]`, `［laughs］` and `[sighs…]` used to * survive into `onTranscript` and memory). Group 1 is the cue. A fresh * RegExp per call: a shared `g` regex carries `lastIndex` between calls. */ function audioTagPattern(): RegExp { return /[[［]\s*([A-Za-z][A-Za-z ,'-]{0,38}?)\s*(?:…|\.{3})?\s*[\]］]/g; } /** Names of the avatar visual tools (for routing + the no-op fallback). */ export const AVATAR_VISUAL_TOOL_NAMES: readonly string[] = AVATAR_VISUAL_TOOLS.map((t) => t.name); /** How a call ended (0.71.0), reported once per call to `onEnd`. * * - `user` — your own `close()`. * - `agent` — the companion ended the call itself: ElevenLabs' built-in * `end_call` system tool. Only on `elevenlabs-convai`, and only when the * agent both has that tool enabled and sends its tool responses to the * client — otherwise the same hang-up arrives as `remote`. * - `remote` — the provider closed the call cleanly without saying why (on * ElevenLabs: the agent left the room, a duration limit, an `end_call` whose * response the client never saw). Not a fault, and not known to be the * companion's decision. * - `error` — a failure; `error` carries it, and `onError` received the same * `Error` first. `openai-realtime` has no clean remote end, so every end * there that is not your `close()` is `error`. */ export interface CallEnd { reason: 'user' | 'agent' | 'remote' | 'error'; error?: Error; } /** Report a call's end. With `onEnd`, a CLEAN remote end (`agent` / `remote`) * goes there only, and `onError` is kept for failures. Without it, every end * that is not the host's own `close()` still reaches `onError` as * `call disconnected`, exactly as before 0.71.0. */ function reportCallEnd(opts: CompanionCallOptions, end: CallEnd): void { if (end.reason === 'error') { if (end.error) opts.onError?.(end.error); opts.onEnd?.(end); return; } if (opts.onEnd) opts.onEnd(end); else if (end.reason !== 'user') opts.onError?.(new CompanionError('call disconnected', 0, 'call_dropped')); } export interface CompanionCallOptions { /** Element to route remote audio to. One is created if omitted. */ audioElement?: HTMLAudioElement; /** One finished line. `text` never carries performed cues. On a call where * `audioTags` was GRANTED (0.70.0), every companion line also carries * `audioTags`: the cues it performed, in order (`[]` when it performed * none). Absent on user lines and on calls without the grant, so absence * means "not applicable", never "zero". */ onTranscript?: (e: { role: 'user' | 'assistant'; text: string; audioTags?: string[] }) => void; onSpeakingChange?: (speaking: boolean) => void; /** A failure that ended the call. Every `onError` is terminal: the call * is already torn down. Without `onEnd`, a clean remote end (the companion * or the provider hanging up) also arrives here as `call disconnected`. */ onError?: (err: Error) => void; /** The call ended, and why (0.71.0). Fires exactly once per call, on * every path including your own `close()`. Passing it moves a clean remote * end (`agent` / `remote`) out of `onError`, so a hang-up is no longer * reported as a fault. See `CallEnd`. */ onEnd?: (end: CallEnd) => void; /** OPT-IN full duplex: keep the mic OPEN while the companion speaks so the * user can interrupt mid-utterance (both providers' native VAD interruption * then works). Default false = half-duplex — the mic is muted during agent * speech, because on phone SPEAKERS browser echo cancellation does not fully * remove the companion's own voice and it barges in on itself (the SDK-voice * "说一句就断了" field report). Enable only where AEC actually holds: * headphones, desktop, or devices you have verified. */ bargeIn?: boolean; /** Open the call with the reader's microphone MUTED (0.69.0); see * `CompanionCall.setMuted`. On `openai-realtime` the track is disabled * before it is attached, so nothing is ever sent. On `elevenlabs-convai` * the mute is applied the moment the session is up; the provider client * has no start-muted option, so the connect handshake itself (before the * agent's first line) can carry audio. */ muted?: boolean; } export interface CompanionCall { /** Hang up. Idempotent. */ close: () => void; /** Inject an out-of-band context line. `speak: true` makes the companion react * now; false stages it for the next turn. * * NOT the reader's words: the line does not reach `onTranscript` or the * memory `/end` consolidates, and `connectCall` uses this same lever for * server-pushed world moments. For something the reader TYPED, use * `sendText`. */ injectEvent: (text: string, speak?: boolean) => void; /** Send a line the READER typed, as their turn in the call (0.69.0). The * companion answers it as something the reader said; it is reported to * `onTranscript` as `role: 'user'` (once, by the SDK, the moment it is * sent) and so reaches the memory `connectCall` hands to `/end` at hang-up, * in order with the spoken lines. * * It takes the floor the way speech does: an utterance the companion is * in the middle of is cut, then it replies to this line. * * Nothing is sent before this call — the provider sees the text only when * you pass it here, so a check you run first runs before anyone reads it. * Returns `false` (and sends nothing) for blank text, after `close()`, or * when the call's channel is not open; `true` once it was handed to the * provider. Same behaviour on both providers. */ sendText: (text: string) => boolean; /** The reader's own mute (0.69.0). `true` stops the microphone sending * audio until `setMuted(false)`. Distinct from the SDK's half-duplex mute * (the mic is muted while the companion speaks, unless `bargeIn`): a * reader who muted stays muted when the companion finishes a line. A * no-op after `close()`. Check `canMute` before showing a mute control. */ setMuted: (muted: boolean) => void; /** The reader's mute as last set (0.69.0) — the `setMuted` / `muted` * state, not whether the half-duplex mute is momentarily engaged. A * method, not a field, so it stays live on a spread copy of the handle. */ isMuted: () => boolean; /** Whether `setMuted` actually controls the microphone on this call * (0.69.0). `true` on `openai-realtime`; on `elevenlabs-convai` it is * `true` when the installed `@elevenlabs/client` exposes `setMicMuted` * (every version the peer range allows does). When `false`, `setMuted` * only records the state: do not show a mute control. */ canMute: boolean; /** SILENT CUT (0.36.0): stop the utterance currently being spoken WITHOUT * generating any new speech — the "stale commentary at a beat boundary" * case (field request: fast-paced game, last hand's commentary still * playing when the next hand starts). Idempotent: a no-op when nothing is * playing, after close(), and mid-connect. * * Provider tiers (check `provider`): * - `openai-realtime` — GUARANTEED: cancels the active response and clears * the WebRTC output buffer (`response.cancel` + `output_audio_buffer.clear`); * playback stops within ~an audio frame. The session continues normally. * - `elevenlabs-convai` — BEST-EFFORT: the EL client exposes no public * "stop playback" API; we signal `user_activity` (halts the agent's * stream when the agent's interruption handling allows) and attempt the * client's internal output-buffer cut when the installed build exposes * it. Verify on your rig; worst case it degrades to a no-op — for a * guaranteed preemption on EL, `injectEvent(text, true)` (which DOES * speak new content) remains the reliable lever. */ interrupt: () => void; provider: CallCredentials['provider']; /** The call's generation handle (0.65.0, set by `connectCall`) — the key * `getCallRecording` / `deleteCallRecording` take. */ callGen?: string; /** Whether this call is recorded (0.65.0) — present when it asked * (`connectCall({ record: true })`). */ recording?: boolean; /** Why a call that asked is not recorded (0.65.0). */ recordingUnavailable?: CallRecordingUnavailable; /** The video avatar's Simli session (0.75.0), when `connectCall({ avatar })` * got one. */ video?: CallVideo; /** Why a call that asked for an avatar has none (0.75.0). */ videoUnavailable?: CallVideoUnavailable; /** Pouchy World role sessions (0.67.0): whether the call carries the * story's current state. See `CallWorldBrief`. */ worldBrief?: CallWorldBrief; /** Pouchy World role sessions (0.68.0): whether the role can reach its * private layer on this call. See `CallPrivateStory`. */ privateStory?: CallPrivateStory; /** What the server cut to fit the realtime cap (0.68.0). See `CallTrimmed`. */ trimmed?: CallTrimmed; } const OPENAI_REALTIME_CALLS = 'https://api.openai.com/v1/realtime/calls'; /** Open a live call from pre-minted credentials. */ export async function openCompanionCall( creds: CallCredentials, opts: CompanionCallOptions = {}, bridge?: VoiceToolBridge ): Promise { if (typeof navigator === 'undefined' || !navigator.mediaDevices) { throw new CompanionError( 'connectCall is browser-only (needs WebRTC + microphone access)', 0, 'call_unsupported' ); } return creds.provider === 'openai-realtime' ? openOpenAICall(creds, opts, bridge) : openConvaiCall(creds, opts, bridge); } // Connect-step timeouts (mirror the first-party realtime path). Without them a // stuck mic (Android WebView holding the device) or a hung SDP exchange hangs // the connect promise forever — and if gUM already resolved, with a hot mic. const GUM_TIMEOUT_MS = 10_000; const SDP_TIMEOUT_MS = 8_000; /** getUserMedia with a timeout. If the timeout wins, a late-resolving mic is * stopped so a slow permission grant can't leave the device recording after * we've already given up on the call. */ async function getMicWithTimeout(): Promise { let timedOut = false; const micPromise = navigator.mediaDevices.getUserMedia({ // Explicit processing constraints: echoCancellation is what makes the // opt-in barge-in mode viable at all, and it is harmless in half-duplex. audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } }); micPromise .then((m) => { if (timedOut) m.getTracks().forEach((t) => t.stop()); }) .catch(() => {}); let timer: ReturnType | undefined; const timeout = new Promise((_, reject) => { timer = setTimeout(() => { timedOut = true; reject(new CompanionError('microphone request timed out', 0, 'call_connect_failed')); }, GUM_TIMEOUT_MS); }); // Clear the timer when the mic wins the race — it held a live 10s closure. return Promise.race([micPromise, timeout]).finally(() => { if (timer) clearTimeout(timer); }); } /** Best-effort read of a FAILED SDP-exchange response body (0.49.0). * * The provider's body is the one place an expired or malformed ephemeral * secret is actually explained, and `connectCall()` is the hardest path in the * SDK to debug from the outside — a bare `SDP exchange failed (401)` is a * status and nothing else. But this runs with the mic ALREADY HOT, so it must * never hang and never throw: * - called before `clearTimeout(sdpTimer)`, so the still-armed abort signal * bounds this read exactly as it bounds the success-path `res.text()` * (the fetch signal aborts body reads too — the 0.38.1 hot-mic doctrine); * - every failure yields '' rather than replacing a real diagnosis with a * body-read error; * - whitespace-collapsed and capped, so a provider HTML error page cannot * flood the thrown message. * * Deliberately module-level rather than inline: `call-sdp-guards.test.ts` pins * the success-path body read by locating the FIRST `await res.text()` inside * `openOpenAICall`, and an inline copy here would capture that pin. */ async function sdpErrorDetail(res: Response): Promise { try { const body = (await res.text()).replace(/\s+/g, ' ').trim(); return body ? ` — ${body.slice(0, 200)}` : ''; } catch { return ''; } } // ── OpenAI Realtime — self-contained WebRTC ───────────────────────────────── async function openOpenAICall( creds: Extract, opts: CompanionCallOptions, bridge?: VoiceToolBridge ): Promise { // Acquire the mic FIRST (with a timeout): if gUM fails/times out there is no // RTCPeerConnection yet to leak. Creating pc before gUM (the old order) leaked // a peer connection on every permission denial. const mic = await getMicWithTimeout(); const pc = new RTCPeerConnection(); const audioEl = opts.audioElement ?? new Audio(); audioEl.autoplay = true; pc.ontrack = (e) => { audioEl.srcObject = e.streams[0]; void audioEl.play().catch(() => {}); }; for (const track of mic.getAudioTracks()) pc.addTrack(track, mic); // Half-duplex mic (mirrors the first-party realtime path, #1524): on a phone // SPEAKER browser AEC doesn't fully cancel the companion's own voice, so it // leaks into the mic, is transcribed as a USER turn, and barges in on itself — // the avatar cuts off mid-sentence after one line (field report: SDK voice on // agentpoker "说一句就断了"). Mute the owned mic track WHILE the model speaks // (sends silence, keeps the track + transport alive), re-open after a short tail // when the turn ends. Tradeoff: no voice barge-in mid-utterance — far better than // the companion interrupting itself. const MIC_UNMUTE_TAIL_MS = 350; let micUnmuteTimer: ReturnType | null = null; // Two independent reasons the mic is off (0.69.0): the READER muted // (setMuted), or the companion is speaking (half-duplex). The track is on // only when neither holds, so the end-of-speech re-open can never undo a // reader's mute — which is what made a mute layered on top of this SDK a // button that switched itself back on. let userMuted = opts.muted === true; let speechMuted = false; const applyMic = () => { try { for (const tr of mic.getAudioTracks()) tr.enabled = !userMuted && !speechMuted; } catch { /* track gone — ignore */ } }; const muteMicForSpeech = () => { if (micUnmuteTimer) { clearTimeout(micUnmuteTimer); micUnmuteTimer = null; } speechMuted = true; applyMic(); }; const scheduleMicUnmute = () => { if (micUnmuteTimer) clearTimeout(micUnmuteTimer); micUnmuteTimer = setTimeout(() => { micUnmuteTimer = null; if (closed) return; speechMuted = false; applyMic(); }, MIC_UNMUTE_TAIL_MS); }; // Whether the model is audibly speaking right now (driven by the // output_audio_buffer events below). Gates interrupt(): cancelling with no // active response makes the server emit a benign-but-noisy error event, so // an idle interrupt() must be a true no-op. let oaiSpeaking = false; // Whether a response is in flight (response.created → response.done) — // broader than `oaiSpeaking`, which starts only with audio. `sendText` // cancels an in-flight response before creating its own: the server // refuses a second concurrent response, which would leave the typed line // in the conversation with no reply. let oaiResponseActive = false; // A reader who asked to start muted never sends a frame: the track is // disabled before the offer is even created. applyMic(); const dc = pc.createDataChannel('oai-events'); // `response.function_call_arguments.done` carries the call_id + args but often // not the name — the name arrives earlier on `response.output_item.added`; track it. // Cancelled tool calls never reach `arguments.done`, so their entries were // never deleted — cap the map (drop-oldest) so a long call can't grow it. const callNames = new Map(); const rememberCallName = (id: string, name: string) => { callNames.set(id, name); if (callNames.size > 64) { const first = callNames.keys().next().value; if (first !== undefined) callNames.delete(first); } }; if (bridge?.tools.length) { dc.addEventListener('open', () => { try { dc.send( JSON.stringify({ type: 'session.update', session: { type: 'realtime', tool_choice: 'auto', tools: bridge.tools.map((t) => ({ type: 'function', name: t.name, description: t.description ?? '', parameters: t.parameters ?? { type: 'object', properties: {} } })) } }) ); } catch { /* a rejected tools update must not kill the call */ } }); } dc.onmessage = (ev) => { let msg: { type?: string; transcript?: string; delta?: string; call_id?: string; name?: string; arguments?: string; item?: { type?: string; name?: string; call_id?: string }; }; try { msg = JSON.parse(ev.data); } catch { return; } // App tool-calling over voice: register name at start, dispatch at done. if ( bridge && msg.type === 'response.output_item.added' && msg.item?.type === 'function_call' && msg.item.call_id && msg.item.name ) { rememberCallName(msg.item.call_id, msg.item.name); } if (bridge && msg.type === 'response.function_call_arguments.done' && msg.call_id) { const callId = msg.call_id; const name = (msg.name && msg.name) || callNames.get(callId) || ''; callNames.delete(callId); if (name && bridge.tools.some((t) => t.name === name)) { const argsText = typeof msg.arguments === 'string' ? msg.arguments : '{}'; // invokeVoiceTool never rejects, but openCompanionCall is a public // export (bring-your-own bridge) — a throwing consumer bridge must // still answer the model, or the voice turn hangs until provider // timeout (plus an unhandled rejection). void bridge .invoke(name, argsText) .catch((e) => JSON.stringify({ ok: false, error: e instanceof Error ? e.message : 'tool failed' }) ) .then((output) => { try { dc.send( JSON.stringify({ type: 'conversation.item.create', item: { type: 'function_call_output', call_id: callId, output } }) ); dc.send(JSON.stringify({ type: 'response.create' })); } catch { /* channel gone */ } }); } return; } if (msg.type === 'response.audio_transcript.done' && msg.transcript) { opts.onTranscript?.({ role: 'assistant', text: msg.transcript }); } else if ( msg.type === 'conversation.item.input_audio_transcription.completed' && msg.transcript ) { opts.onTranscript?.({ role: 'user', text: msg.transcript }); } else if (msg.type === 'response.created') { oaiResponseActive = true; } else if (msg.type === 'output_audio_buffer.started') { oaiSpeaking = true; opts.onSpeakingChange?.(true); // Model started speaking → mute the mic so its own audio can't // echo-transcribe into a self-barge-in. bargeIn opts out (B wave): // the mic stays open and OpenAI's server VAD handles interruption. if (opts.bargeIn !== true) muteMicForSpeech(); } else if ( msg.type === 'output_audio_buffer.stopped' || msg.type === 'output_audio_buffer.cleared' || msg.type === 'response.done' || msg.type === 'conversation.interrupted' ) { // Turn ended (or was cancelled / interrupted) — clear the speaking state // and re-open the mic after a short tail so the last bit of the model's // audio doesn't get captured the instant we unmute. Several event names // can end a turn; any of them re-arms the mic. oaiSpeaking = false; if (msg.type === 'response.done') oaiResponseActive = false; opts.onSpeakingChange?.(false); if (opts.bargeIn !== true) scheduleMicUnmute(); } }; const offer = await pc.createOffer(); await pc.setLocalDescription(offer); // Bound the SDP exchange so a hung network can't leave the pc + hot mic open // forever. On any failure, close both before throwing. const sdpAbort = new AbortController(); const sdpTimer = setTimeout(() => sdpAbort.abort(), SDP_TIMEOUT_MS); const cleanupPartial = () => { pc.close(); mic.getTracks().forEach((t) => t.stop()); }; let res: Response; try { res = await fetch(OPENAI_REALTIME_CALLS, { method: 'POST', headers: { Authorization: `Bearer ${creds.clientSecret}`, 'Content-Type': 'application/sdp' }, body: offer.sdp, signal: sdpAbort.signal }); } catch { clearTimeout(sdpTimer); cleanupPartial(); throw new CompanionError( sdpAbort.signal.aborted ? 'OpenAI Realtime SDP exchange timed out' : 'OpenAI Realtime SDP exchange failed', 0, 'call_connect_failed' ); } if (!res.ok) { // Read the provider's explanation BEFORE clearing the timer, so the same // signal that bounds the success-path body read bounds this one. const detail = await sdpErrorDetail(res); clearTimeout(sdpTimer); cleanupPartial(); throw new CompanionError( `OpenAI Realtime SDP exchange failed (${res.status})${detail}`, res.status, 'call_connect_failed' ); } // The answer BODY read must stay under the sdp timer too: clearing the timer // at headers left `res.text()` unbounded — a mid-body stall hung connectCall() // forever with the mic already hot (the exact hazard the timer exists for). // The fetch signal aborts body reads as well, so the same timer covers both. let answerSdp: string; try { answerSdp = await res.text(); } catch { cleanupPartial(); throw new CompanionError( sdpAbort.signal.aborted ? 'OpenAI Realtime SDP exchange timed out' : 'OpenAI Realtime SDP exchange failed', 0, 'call_connect_failed' ); } finally { clearTimeout(sdpTimer); } try { await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp }); } catch { // Malformed/rejected answer — without this, the raw DOMException escaped // un-wrapped and the pc + mic tracks leaked. cleanupPartial(); throw new CompanionError( 'OpenAI Realtime SDP answer rejected', 0, 'call_connect_failed' ); } let closed = false; // `disconnected` is TRANSIENT per the WebRTC spec — a brief Wi-Fi hiccup or a // cell handoff routinely drops to `disconnected` and recovers to `connected` // on its own. Tearing down the instant it appears killed calls that would have // healed (field report: SDK voice "动不动就掉线"). Give it a grace window to // come back; only if it's STILL disconnected when the timer fires do we tear // down — so the mic still can't record forever on a genuine silent drop (the // original concern). `failed` / `closed` stay immediately fatal. const DISCONNECT_GRACE_MS = 6000; let disconnectTimer: ReturnType | null = null; const clearDisconnectTimer = () => { if (disconnectTimer) { clearTimeout(disconnectTimer); disconnectTimer = null; } }; const teardown = () => { if (closed) return; closed = true; clearDisconnectTimer(); if (micUnmuteTimer) { clearTimeout(micUnmuteTimer); micUnmuteTimer = null; } try { dc.close(); } catch { /* ignore */ } pc.close(); mic.getTracks().forEach((t) => t.stop()); }; pc.onconnectionstatechange = () => { const st = pc.connectionState; if (st === 'connected') { // (Re)connected — cancel any pending disconnect teardown. clearDisconnectTimer(); return; } if (st === 'failed' || st === 'closed') { // Genuinely dead — tear down immediately, BEFORE surfacing the error, so a // silent transport drop can't leave the mic recording (mirrors the // first-party realtime/connection.ts fail() ordering). clearDisconnectTimer(); teardown(); reportCallEnd(opts, { reason: 'error', error: new CompanionError(`call ${st}`, 0, 'call_dropped') }); return; } if (st === 'disconnected') { // Hold a grace window; a recovery to `connected` (handled above) cancels // the timer. Still disconnected when it fires → tear down. if (disconnectTimer || closed) return; disconnectTimer = setTimeout(() => { disconnectTimer = null; if (closed) return; if (pc.connectionState === 'disconnected') { teardown(); reportCallEnd(opts, { reason: 'error', error: new CompanionError('call disconnected', 0, 'call_dropped') }); } }, DISCONNECT_GRACE_MS); } }; // Data-channel liveness (mirrors the first-party dataChannel.onclose/onerror). // A dc-only death — the channel closes/errors while pc.connectionState still // reads 'connected' — otherwise goes undetected and the mic stays hot. // teardown flips `closed` before it calls dc.close(), so a normal teardown's // own channel close is guarded out here (no double-fire / spurious error). dc.onclose = () => { if (closed) return; teardown(); reportCallEnd(opts, { reason: 'error', error: new CompanionError('call data channel closed', 0, 'call_dropped') }); }; dc.onerror = () => { if (closed) return; teardown(); reportCallEnd(opts, { reason: 'error', error: new CompanionError('call data channel error', 0, 'call_dropped') }); }; return { provider: 'openai-realtime', close: () => { if (closed) return; teardown(); reportCallEnd(opts, { reason: 'user' }); }, injectEvent: (text, speak = false) => { if (closed || dc.readyState !== 'open' || !text.trim()) return; dc.send( JSON.stringify({ type: 'conversation.item.create', item: { type: 'message', role: 'user', content: [{ type: 'input_text', text }] } }) ); if (speak) dc.send(JSON.stringify({ type: 'response.create' })); }, sendText: (text) => { const line = typeof text === 'string' ? text.trim() : ''; if (closed || dc.readyState !== 'open' || !line) return false; try { if (oaiResponseActive) { dc.send(JSON.stringify({ type: 'response.cancel' })); if (oaiSpeaking) dc.send(JSON.stringify({ type: 'output_audio_buffer.clear' })); } dc.send( JSON.stringify({ type: 'conversation.item.create', item: { type: 'message', role: 'user', content: [{ type: 'input_text', text: line }] } }) ); dc.send(JSON.stringify({ type: 'response.create' })); } catch { return false; } // A typed item produces no input-audio transcription event, so the SDK // reports it itself — that is what puts it in the caller's transcript // and in the memory connectCall consolidates at hang-up. opts.onTranscript?.({ role: 'user', text: line }); return true; }, setMuted: (muted) => { userMuted = muted === true; if (!closed) applyMic(); }, isMuted: () => userMuted, canMute: true, interrupt: () => { // Silent cut: stop generating (response.cancel) AND flush what's // already buffered for playback (output_audio_buffer.clear — the // WebRTC-mode control for exactly this). Order matters: cancel first // so no new audio lands behind the clear. The `.cleared` event echoes // back through the handler above, which resets the speaking state and // re-arms the mic like any natural end of turn. if (closed || !oaiSpeaking || dc.readyState !== 'open') return; try { dc.send(JSON.stringify({ type: 'response.cancel' })); dc.send(JSON.stringify({ type: 'output_audio_buffer.clear' })); } catch { /* channel gone — the call is tearing down anyway */ } } }; } // ── ElevenLabs Convai — via the optional @elevenlabs/client peer dep ───────── /** Map the EL client's `DisconnectionDetails` onto `CallEnd`. An absent or * unrecognised shape is a failure, never a clean end: unknown is not the * benign answer. */ export function convaiCallEnd(details?: { reason?: string; message?: string; context?: { type?: string }; }): CallEnd { if (details?.reason === 'agent') { return { reason: details.context?.type === 'end_call' ? 'agent' : 'remote' }; } // `user` here is the EL client ending its own session without our close() // (ours sets `closed` first and never reaches this) — a clean far-side end. if (details?.reason === 'user') return { reason: 'remote' }; // The provider NAMED a failure → `call_provider_error`; anything else is the // line going away → `call_dropped` (CR-974: switchable, not English text). if (details?.reason === 'error' && typeof details.message === 'string' && details.message) { return { reason: 'error', error: new CompanionError(details.message, 0, 'call_provider_error') }; } return { reason: 'error', error: new CompanionError('call disconnected', 0, 'call_dropped') }; } async function openConvaiCall( creds: Extract, opts: CompanionCallOptions, bridge?: VoiceToolBridge ): Promise { let mod: typeof import('@elevenlabs/client'); try { mod = await import('@elevenlabs/client'); } catch { throw new CompanionError( "elevenlabs-convai call needs the optional peer dependency '@elevenlabs/client' — install it (npm i @elevenlabs/client)", 0, 'call_dependency_missing' ); } // App tools → EL client tools. NOTE: ElevenLabs only CALLS a client tool the // agent already knows (declared on the agent config); a shared Convai agent // can't take arbitrary per-app tools at runtime, so dynamic app tools are // reliable on the OpenAI Realtime path. Registering the handlers is harmless // and works for any tool the agent does know. const clientTools: Record Promise> = {}; if (bridge?.tools.length) { for (const t of bridge.tools) { clientTools[t.name] = (params: unknown) => bridge.invoke(t.name, JSON.stringify(params ?? {})); } } let closed = false; // Whether the agent is audibly speaking (onModeChange) — gates interrupt() // so an idle call stays a true no-op. let elSpeaking = false; // Half-duplex mic (mirrors the first-party path, #1524): mute while the agent // speaks so its own voice can't echo-transcribe into a self-barge-in (the SDK // voice "说一句就断了" report), re-open after a short tail. Convai exposes // setMicMuted; guard it (older client builds may lack it). const MIC_UNMUTE_TAIL_MS = 350; let micUnmuteTimer: ReturnType | null = null; const setConvMicMuted = (muted: boolean) => { try { (conv as unknown as { setMicMuted?: (m: boolean) => void }).setMicMuted?.(muted); } catch { /* not supported on this client build — ignore */ } }; // The reader's mute and the half-duplex mute are separate reasons (0.69.0, // see the OpenAI leg); the mic opens only when neither holds. let userMuted = opts.muted === true; let speechMuted = false; const applyMic = () => setConvMicMuted(userMuted || speechMuted); // Lines `sendText` already reported. Should the provider ever echo a typed // user_message back as a user transcript, it must not be reported twice // (and land in memory twice). Matched by exact text, consumed once. const typedEcho: string[] = []; const conv = await mod.Conversation.startSession({ conversationToken: creds.token, connectionType: 'webrtc', ...(Object.keys(clientTools).length ? { clientTools } : {}), // The credential the agent's memory WEBHOOK tool presents back to us // mid-call. It is a dynamic variable rather than anything the model can // see: EL substitutes it into the tool's Authorization header, and the // `secret__` class is never sent to the LLM. // // Omitted when the mint returned none (a session with no memory grant) — // the agent config carries a default, so the tool then reaches us with a // credential that cannot verify, we 401, and the model reads a typed // refusal. That is a degraded recall, not a failed call. ...(creds.voiceToolToken ? { dynamicVariables: { // BOTH names. EL delivered the `secret__` one unsubstituted in // four field readings — that class is designed to ignore a // CLIENT-supplied value — so the plain twin is what is expected // to arrive. Sending both keeps the secret path working if EL // ever honours it, and the endpoint records which one did. [VOICE_TOOL_VARIABLE]: creds.voiceToolToken, [VOICE_TOOL_VARIABLE_PLAIN]: creds.voiceToolToken } } : {}), overrides: { agent: { // The server-assembled persona + memory snapshot drives the EL agent. prompt: { prompt: creds.instructions }, ...(creds.language ? { language: creds.language as never } : {}), ...(creds.firstMessage !== undefined ? { firstMessage: creds.firstMessage } : {}) }, ...(creds.voice ? { tts: { voiceId: creds.voice } } : {}) }, onModeChange: ({ mode }: { mode: string }) => { if (closed) return; const speaking = mode === 'speaking'; elSpeaking = speaking; opts.onSpeakingChange?.(speaking); // bargeIn opts out of half-duplex (B wave): the mic stays open and // ElevenLabs' native interruption handling takes over. if (opts.bargeIn === true) return; if (speaking) { if (micUnmuteTimer) { clearTimeout(micUnmuteTimer); micUnmuteTimer = null; } speechMuted = true; applyMic(); } else { if (micUnmuteTimer) clearTimeout(micUnmuteTimer); micUnmuteTimer = setTimeout(() => { micUnmuteTimer = null; if (closed) return; speechMuted = false; applyMic(); }, MIC_UNMUTE_TAIL_MS); } }, onMessage: ({ message, role }: { message: string; role: string }) => { // Performed cues are voice direction, not words — only present when // the server granted `audioTags` for this call (0.64.0). const raw = (message ?? '').trim(); const text = creds.audioTags && role === 'agent' ? stripAudioTags(raw) : raw; // What was performed, read off the same raw line before it is // stripped (0.70.0): the transcript and memory stay clean, and the // integrator can still count the cues. const cues = creds.audioTags && role === 'agent' ? { audioTags: extractAudioTags(raw) } : {}; if (!closed && text) { if (role !== 'agent') { const echo = typedEcho.indexOf(text); if (echo !== -1) { typedEcho.splice(echo, 1); return; } } opts.onTranscript?.({ role: role === 'agent' ? 'assistant' : 'user', text, ...cues }); } }, onError: (message: string) => { if (closed) return; // End the session (releases the EL SDK's mic + hidden