import { createContext, useCallback, useContext, useEffect, useMemo, useRef, useState, type ReactNode, } from "react"; import { createPortal } from "react-dom"; import { createRealtimeVoiceAudioLevelStore, normalizeRealtimeVoiceRms, smoothRealtimeVoiceLevel, type RealtimeVoiceAudioLevelStore, } from "./realtime-voice-audio-level.js"; import { realtimeVoiceTranscriptRegistry } from "./realtime-voice-transcript.js"; import { RealtimeVoiceModeDock, type RealtimeVoiceModeCopy, type RealtimeVoiceModeState, } from "./RealtimeVoiceMode.js"; import { useComposerRuntimeAdapters } from "./runtime-adapters.js"; const REALTIME_VOICE_STATE_KEY = "realtime-voice-session"; const REALTIME_VOICE_PREFERENCES_KEY = "realtime-voice-prefs"; const REALTIME_VOICE_REQUEST_SOURCE = "realtime-voice"; const REALTIME_VOICE_SESSION_PATH = "/_agent-native/realtime-voice/session"; const REALTIME_VOICE_TOOL_PATH = "/_agent-native/realtime-voice/tool"; const REALTIME_VOICE_CAPABILITY_HEADER = "X-Agent-Native-Realtime-Capability"; const REALTIME_VOICE_CONNECTION_TIMEOUT_MS = 15_000; const REALTIME_VOICE_MAX_TOOLS = 32; const REALTIME_VOICE_MAX_TOOL_SCHEMA_BYTES = 32_000; const REALTIME_VOICE_MAX_SESSION_BYTES = 64_000; const REALTIME_VOICE_TOOL_UPDATE_TIMEOUT_MS = 5_000; const REALTIME_VOICE_PRIORITY_TOOL_NAMES = [ "navigate", "set-url-path", "set-search-params", "view-screen", "tool-search", ] as const; const REALTIME_VOICE_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/; const REALTIME_VOICE_MICROPHONE_STORAGE_KEY = "agent-native:realtime-voice-microphone"; export const REALTIME_VOICE_LANGUAGES = [ "auto", "en", "es", "fr", "de", "it", "pt", "ja", "ko", "zh", ] as const; export const REALTIME_VOICE_INTELLIGENCE_LEVELS = [ "instant", "balanced", "deep", ] as const; export const REALTIME_VOICES = [ "marin", "cedar", "coral", "sage", "verse", "alloy", "ash", "ballad", "echo", "shimmer", ] as const; export type RealtimeVoiceLanguage = (typeof REALTIME_VOICE_LANGUAGES)[number]; export type RealtimeVoiceIntelligence = (typeof REALTIME_VOICE_INTELLIGENCE_LEVELS)[number]; export type RealtimeVoice = (typeof REALTIME_VOICES)[number]; export interface RealtimeVoicePreferences { language: RealtimeVoiceLanguage; intelligence: RealtimeVoiceIntelligence; voice: RealtimeVoice; } export const DEFAULT_REALTIME_VOICE_PREFERENCES: RealtimeVoicePreferences = { language: "auto", intelligence: "instant", voice: "marin", }; export const REALTIME_VOICE_AUDIO_CONSTRAINTS: MediaTrackConstraints = { // Prefer the browser/OS-selected input instead of whichever physical device // happens to be enumerated first. `ideal` keeps browsers without the // synthetic `default` device from failing with OverconstrainedError. deviceId: { ideal: "default" }, echoCancellation: true, noiseSuppression: true, autoGainControl: true, }; type RealtimeServerEvent = Record & { type?: string }; export interface RealtimeVoiceToolResult { callId: string; status: "completed" | "failed" | "approval_required"; output: string; approvalKey?: string; expandedTools?: RealtimeVoiceFunctionTool[]; } export interface RealtimeVoiceSessionAnswer { sdp: string; capability?: string; } export interface RealtimeVoiceFunctionTool { type: "function"; name: string; description?: string; parameters?: Record; } export interface RealtimeVoiceModeApi { state: "idle" | RealtimeVoiceModeState; active: boolean; errorMessage: string | null; chatVisible: boolean; audioLevels: RealtimeVoiceAudioLevelStore; preferences: RealtimeVoicePreferences; microphones: readonly RealtimeVoiceMicrophone[]; microphoneDeviceId: string; microphoneSwitching: boolean; microphoneError: string | null; voiceChangePending: boolean; setLanguage: (language: RealtimeVoiceLanguage) => void; setIntelligence: (intelligence: RealtimeVoiceIntelligence) => void; setVoice: (voice: RealtimeVoice) => void; setMicrophone: (deviceId: string) => Promise; start: () => Promise; end: () => void; toggleChat: () => void; } export interface RealtimeVoiceMicrophone { deviceId: string; label: string; } export function createRealtimeVoiceAudioConstraints( deviceId = "default", exact = false, ): MediaTrackConstraints { return { ...REALTIME_VOICE_AUDIO_CONSTRAINTS, deviceId: deviceId === "default" ? { ideal: "default" } : exact ? { exact: deviceId } : { ideal: deviceId }, }; } export async function replaceRealtimeVoiceMicrophone(options: { mediaDevices: Pick; peer: Pick; currentStream: MediaStream | null; deviceId: string; }): Promise { const replacementStream = await options.mediaDevices.getUserMedia({ audio: createRealtimeVoiceAudioConstraints(options.deviceId, true), }); try { const replacementTrack = replacementStream.getAudioTracks()[0]; const sender = options.peer .getSenders() .find((candidate) => candidate.track?.kind === "audio"); if (!replacementTrack || !sender) { throw new Error("No active microphone track is available."); } await sender.replaceTrack(replacementTrack); for (const track of options.currentStream?.getTracks() ?? []) { track.onended = null; track.stop(); } return replacementStream; } catch (error) { for (const track of replacementStream.getTracks()) track.stop(); throw error; } } function readRealtimeVoiceMicrophoneId(): string { if (typeof window === "undefined") return "default"; try { return ( window.localStorage.getItem(REALTIME_VOICE_MICROPHONE_STORAGE_KEY) || "default" ); } catch { return "default"; } } function writeRealtimeVoiceMicrophoneId(deviceId: string): void { try { window.localStorage.setItem( REALTIME_VOICE_MICROPHONE_STORAGE_KEY, deviceId, ); } catch { // A remembered local device is optional and must never block voice mode. } } function isOneOf( values: T, value: unknown, ): value is T[number] { return typeof value === "string" && values.includes(value); } export function normalizeRealtimeVoicePreferences( value: unknown, ): RealtimeVoicePreferences { const record = value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; return { language: isOneOf(REALTIME_VOICE_LANGUAGES, record.language) ? record.language : DEFAULT_REALTIME_VOICE_PREFERENCES.language, intelligence: isOneOf( REALTIME_VOICE_INTELLIGENCE_LEVELS, record.intelligence, ) ? record.intelligence : DEFAULT_REALTIME_VOICE_PREFERENCES.intelligence, voice: isOneOf(REALTIME_VOICES, record.voice) ? record.voice : DEFAULT_REALTIME_VOICE_PREFERENCES.voice, }; } export function resolveRealtimeVoiceLanguage( language: RealtimeVoiceLanguage, browserLanguages: readonly string[] = [], ): Exclude { if (language !== "auto") return language; for (const locale of browserLanguages) { const primary = locale.trim().split("-")[0]?.toLowerCase(); if (isOneOf(REALTIME_VOICE_LANGUAGES, primary) && primary !== "auto") { return primary; } } return "en"; } export function realtimeVoiceReasoningEffort( intelligence: RealtimeVoiceIntelligence, ): "minimal" | "low" | "medium" { if (intelligence === "balanced") return "low"; if (intelligence === "deep") return "medium"; return "minimal"; } export function createRealtimeVoicePreferenceUpdate( preferences: RealtimeVoicePreferences, options: { browserLanguages?: readonly string[]; includeVoice?: boolean; } = {}, ): Record { return { type: "session.update", session: { type: "realtime", reasoning: { effort: realtimeVoiceReasoningEffort(preferences.intelligence), }, audio: { input: { transcription: { model: "gpt-4o-mini-transcribe", language: resolveRealtimeVoiceLanguage( preferences.language, options.browserLanguages, ), }, }, ...(options.includeVoice ? { output: { voice: preferences.voice } } : {}), }, }, }; } export interface CompletedRealtimeVoiceTranscript { role: "user" | "assistant"; text: string; providerId?: string; } export interface RealtimeVoiceTranscriptSequencer { handle: (event: RealtimeServerEvent) => void; reset: () => void; } interface SequencedRealtimeVoiceTranscript { status: "pending" | "completed" | "skipped"; role?: CompletedRealtimeVoiceTranscript["role"]; transcript?: CompletedRealtimeVoiceTranscript; } /** * Input transcription is produced by a separate ASR model and can finish after * the assistant response. Reserve each message's position when OpenAI adds it * to the conversation, then publish only the contiguous completed prefix. */ export function createRealtimeVoiceTranscriptSequencer( publish: (transcript: CompletedRealtimeVoiceTranscript) => void, ): RealtimeVoiceTranscriptSequencer { const order: string[] = []; const items = new Map(); const settledItemIds = new Set(); const receivedTranscriptIds = new Set(); const reserve = ( id: string, role?: CompletedRealtimeVoiceTranscript["role"], ) => { if (settledItemIds.has(id)) return; const existing = items.get(id); if (existing) { existing.role ??= role; return; } order.push(id); items.set(id, { status: "pending", role }); }; const drain = () => { while (order.length > 0) { const id = order[0]!; const entry = items.get(id); if (!entry || entry.status === "pending") return; order.shift(); items.delete(id); settledItemIds.add(id); if (entry.status === "completed" && entry.transcript) { publish(entry.transcript); } } }; const skip = (id: unknown) => { if (typeof id !== "string" || !id) return; reserve(id); const entry = items.get(id)!; if (entry.status === "pending") entry.status = "skipped"; }; return { handle(event) { if ( event.type === "conversation.item.added" || event.type === "conversation.item.created" ) { const item = event.item; if (item && typeof item === "object") { const record = item as Record; if ( record.type === "message" && (record.role === "user" || record.role === "assistant") && typeof record.id === "string" ) { reserve(record.id, record.role); } } } const transcript = extractCompletedRealtimeVoiceTranscript(event); if (transcript) { const transcriptId = transcript.providerId ? `${transcript.role}:${transcript.providerId}` : undefined; if (transcriptId && receivedTranscriptIds.has(transcriptId)) return; if (transcriptId) receivedTranscriptIds.add(transcriptId); const itemId = typeof event.item_id === "string" ? event.item_id : undefined; if (!itemId) { // Older providers may omit item_id. Match the first reserved slot for // that role so one incomplete legacy event cannot strand the queue. const reservedId = order.find((id) => { const entry = items.get(id); return ( entry?.status === "pending" && entry.role === transcript.role ); }); if (reservedId) { items.set(reservedId, { status: "completed", role: transcript.role, transcript, }); } else { publish(transcript); } } else { if (settledItemIds.has(itemId)) return; reserve(itemId, transcript.role); items.set(itemId, { status: "completed", role: transcript.role, transcript, }); } } else if ( event.type === "conversation.item.input_audio_transcription.failed" || event.type === "conversation.item.deleted" ) { skip(event.item_id); } else if (event.type === "response.done") { const response = event.response; const output = response && typeof response === "object" ? (response as { output?: unknown }).output : undefined; if (Array.isArray(output)) { for (const item of output) { if (!item || typeof item !== "object") continue; const record = item as Record; if (record.type === "message") skip(record.id); } } } drain(); }, reset() { order.length = 0; items.clear(); settledItemIds.clear(); receivedTranscriptIds.clear(); }, }; } export function createRealtimeVoiceGreetingEvent(): Record { return { type: "response.create", response: { output_modalities: ["audio"], instructions: 'Say exactly: "How can I help you?" Do not add anything else.', // Do not set max_output_tokens here. Realtime counts assistant audio at // roughly one token per 50 ms, so a text-sized cap can cut speech off. // The exact-response instruction already keeps this greeting bounded. }, }; } export function createRealtimeVoiceGreetingStarter( send: (event: Record) => void, ): { start: () => boolean; reset: () => void } { let started = false; return { start() { if (started) return false; started = true; send(createRealtimeVoiceGreetingEvent()); return true; }, reset() { started = false; }, }; } export interface RealtimeVoiceResponseCoordinator { request: (event?: Record) => void; handleEvent: (event: RealtimeServerEvent) => void; handleError: (message?: string) => boolean; reset: () => void; } export function createRealtimeVoiceResponseCoordinator( send: (event: Record) => void, ): RealtimeVoiceResponseCoordinator { let active = false; let requestInFlight = false; let pending: Record | undefined; const flush = () => { if (active || requestInFlight || !pending) return; const event = pending; pending = undefined; requestInFlight = true; send(event); }; return { request(event = { type: "response.create" }) { pending = event; flush(); }, handleEvent(event) { if (event.type === "response.created") { active = true; requestInFlight = false; return; } if ( event.type === "response.done" || event.type === "response.cancelled" || event.type === "response.failed" ) { active = false; requestInFlight = false; const response = event.response; const status = response && typeof response === "object" ? (response as { status?: unknown }).status : undefined; if (event.type === "response.failed" || status === "failed") { pending = undefined; return; } flush(); } }, handleError(message) { if (!message || !/active response.*in progress/i.test(message)) { return false; } pending ??= { type: "response.create" }; requestInFlight = false; active = true; return true; }, reset() { active = false; requestInFlight = false; pending = undefined; }, }; } /** * Voice mode owns the chat only temporarily. Restore the captured transcript * when it is still the user's active thread (or the chat has no active thread) * but never pull them back after they deliberately selected another thread. */ export function shouldRestoreRealtimeVoiceTranscriptThread( transcriptThreadId: string | undefined, activeThreadId: string | undefined, ): transcriptThreadId is string { return Boolean( transcriptThreadId && (!activeThreadId || activeThreadId === transcriptThreadId), ); } export function extractCompletedRealtimeVoiceTranscript( event: RealtimeServerEvent, ): CompletedRealtimeVoiceTranscript | null { const userCompleted = event.type === "conversation.item.input_audio_transcription.completed"; const assistantCompleted = event.type === "response.output_audio_transcript.done"; if (!userCompleted && !assistantCompleted) return null; const text = typeof event.transcript === "string" ? event.transcript.trim() : ""; if (!text) return null; const providerId = [event.item_id, event.response_id, event.event_id].find( (value): value is string => typeof value === "string" && value.length > 0, ); return { role: userCompleted ? "user" : "assistant", text, ...(providerId ? { providerId } : {}), }; } export interface RealtimeVoiceModeProviderProps { children: ReactNode; browserTabId?: string; } function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } export function isRealtimeVoiceAbortError(error: unknown): boolean { return Boolean( error && typeof error === "object" && (error as { name?: unknown }).name === "AbortError", ); } export function isRealtimeVoiceSetupRequiredError(error: unknown): boolean { return Boolean( error && typeof error === "object" && (error as { status?: unknown }).status === 409, ); } export function createRealtimeVoiceConnectionTimeout( onTimeout: () => void, timeoutMs = REALTIME_VOICE_CONNECTION_TIMEOUT_MS, ): () => void { const timeout = window.setTimeout(onTimeout, timeoutMs); return () => window.clearTimeout(timeout); } export function createRealtimeVoiceConnectionGate( onTimeout: () => void, timeoutMs = REALTIME_VOICE_CONNECTION_TIMEOUT_MS, ): { markTransportReady: () => void; markSessionCreated: () => void; cancel: () => void; } { const cancel = createRealtimeVoiceConnectionTimeout(onTimeout, timeoutMs); return { // RTC connectivity alone is not enough: the session may never finish its // Realtime handshake. Keep the deadline armed until session.created. markTransportReady() {}, markSessionCreated: cancel, cancel, }; } async function readErrorResponse(response: Response): Promise { const raw = await response.text().catch(() => ""); if (!raw) return response.statusText || `HTTP ${response.status}`; try { const parsed = JSON.parse(raw) as { error?: unknown; message?: unknown }; return String(parsed.error ?? parsed.message ?? raw); } catch { return raw.slice(0, 500); } } interface RealtimeVoiceSessionOptions { browserTabId?: string; signal?: AbortSignal; preferences?: RealtimeVoicePreferences; browserLanguages?: readonly string[]; } export async function createRealtimeVoiceSessionWithCapability( offerSdp: string, options: RealtimeVoiceSessionOptions = {}, ): Promise { const preferences = options.preferences; const response = await fetch(REALTIME_VOICE_SESSION_PATH, { method: "POST", credentials: "same-origin", headers: { "Content-Type": "application/sdp", ...(options.browserTabId ? { "X-Agent-Native-Browser-Tab": options.browserTabId } : {}), ...(preferences ? { "X-Agent-Native-Realtime-Language": resolveRealtimeVoiceLanguage( preferences.language, options.browserLanguages, ), "X-Agent-Native-Realtime-Intelligence": preferences.intelligence, "X-Agent-Native-Realtime-Voice": preferences.voice, } : {}), }, body: offerSdp, signal: options.signal, }); if (!response.ok) { const message = await readErrorResponse(response); const error = new Error(message); (error as { status?: number }).status = response.status; throw error; } const sdp = await response.text(); const capability = response.headers.get(REALTIME_VOICE_CAPABILITY_HEADER); return { sdp, ...(capability ? { capability } : {}), }; } /** Backwards-compatible SDP-only session helper for existing client callers. */ export async function createRealtimeVoiceSession( offerSdp: string, options: RealtimeVoiceSessionOptions = {}, ): Promise { return (await createRealtimeVoiceSessionWithCapability(offerSdp, options)) .sdp; } export async function executeRealtimeVoiceTool(input: { name: string; args: Record; callId: string; sessionId?: string; browserTabId?: string; capability?: string; signal?: AbortSignal; }): Promise { const response = await fetch(REALTIME_VOICE_TOOL_PATH, { method: "POST", credentials: "same-origin", headers: { "Content-Type": "application/json", ...(input.browserTabId ? { "X-Agent-Native-Browser-Tab": input.browserTabId } : {}), ...(input.capability ? { [REALTIME_VOICE_CAPABILITY_HEADER]: input.capability } : {}), }, body: JSON.stringify({ name: input.name, args: input.args, callId: input.callId, sessionId: input.sessionId, browserTabId: input.browserTabId, }), signal: input.signal, }); if (!response.ok) { throw new Error(await readErrorResponse(response)); } return (await response.json()) as RealtimeVoiceToolResult; } function jsonByteLength(value: unknown): number { return new TextEncoder().encode(JSON.stringify(value)).byteLength; } function normalizeRealtimeVoiceFunctionTool( value: unknown, ): RealtimeVoiceFunctionTool | null { if (!value || typeof value !== "object" || Array.isArray(value)) return null; const record = value as Record; if ( record.type !== "function" || typeof record.name !== "string" || !REALTIME_VOICE_TOOL_NAME.test(record.name) ) { return null; } if ( record.description !== undefined && typeof record.description !== "string" ) { return null; } if ( record.parameters !== undefined && (!record.parameters || typeof record.parameters !== "object" || Array.isArray(record.parameters)) ) { return null; } try { if (jsonByteLength(record) > REALTIME_VOICE_MAX_TOOL_SCHEMA_BYTES) { return null; } return JSON.parse(JSON.stringify(record)) as RealtimeVoiceFunctionTool; } catch { return null; } } export function extractRealtimeVoiceSessionTools( event: RealtimeServerEvent, ): RealtimeVoiceFunctionTool[] | null { if (event.type !== "session.created" && event.type !== "session.updated") { return null; } const session = event.session; if (!session || typeof session !== "object") return null; const tools = (session as { tools?: unknown }).tools; if (!Array.isArray(tools)) return null; return tools .map(normalizeRealtimeVoiceFunctionTool) .filter((tool): tool is RealtimeVoiceFunctionTool => Boolean(tool)); } function createRealtimeVoiceToolManifestUpdate( tools: RealtimeVoiceFunctionTool[], eventId?: string, ): Record { return { type: "session.update", ...(eventId ? { event_id: eventId } : {}), session: { type: "realtime", tools, tool_choice: "auto", }, }; } /** * Merge search-discovered schemas into the live manifest. Pinned navigation * and discovery tools stay available, then the newest discoveries take the * remaining slots ahead of lower-priority tools from the original manifest. */ export function mergeRealtimeVoiceToolManifest( currentTools: readonly RealtimeVoiceFunctionTool[], expandedTools: readonly RealtimeVoiceFunctionTool[], ): RealtimeVoiceFunctionTool[] { const current = new Map(); for (const value of currentTools) { const tool = normalizeRealtimeVoiceFunctionTool(value); if (tool) current.set(tool.name, tool); } const expanded = new Map(); for (const value of expandedTools) { const tool = normalizeRealtimeVoiceFunctionTool(value); if (tool) expanded.set(tool.name, tool); } const candidates: RealtimeVoiceFunctionTool[] = []; const seen = new Set(); const add = (tool: RealtimeVoiceFunctionTool | undefined) => { if (!tool || seen.has(tool.name)) return; seen.add(tool.name); candidates.push(tool); }; for (const name of REALTIME_VOICE_PRIORITY_TOOL_NAMES) { add(expanded.get(name) ?? current.get(name)); } for (const tool of expanded.values()) add(tool); for (const tool of current.values()) add(tool); const packed: RealtimeVoiceFunctionTool[] = []; for (const tool of candidates) { if (packed.length >= REALTIME_VOICE_MAX_TOOLS) break; const candidate = [...packed, tool]; if ( jsonByteLength( createRealtimeVoiceToolManifestUpdate( candidate, "realtime_tool_manifest_999999999", ), ) <= REALTIME_VOICE_MAX_SESSION_BYTES ) { packed.push(tool); } } return packed; } function createRealtimeVoiceToolResultEvent( result: RealtimeVoiceToolResult, ): Record { const modelResult = { callId: result.callId, status: result.status, output: result.output, ...(result.approvalKey ? { approvalKey: result.approvalKey } : {}), }; return { type: "conversation.item.create", item: { type: "function_call_output", call_id: result.callId, output: JSON.stringify(modelResult), }, }; } export interface RealtimeVoiceToolManifestCoordinator { enqueue: (result: RealtimeVoiceToolResult) => void; setSessionTools: (tools: readonly RealtimeVoiceFunctionTool[]) => void; handleError: (eventId: string | undefined, message?: string) => boolean; reset: () => void; getTools: () => readonly RealtimeVoiceFunctionTool[]; } /** * Realtime treats `session.update.tools` as a full replacement. Serialize * discovery updates and wait for a confirming `session.updated` manifest * before returning the search result and allowing the next model response. */ export function createRealtimeVoiceToolManifestCoordinator( send: (event: Record) => void, timeoutMs = REALTIME_VOICE_TOOL_UPDATE_TIMEOUT_MS, ): RealtimeVoiceToolManifestCoordinator { let currentTools: RealtimeVoiceFunctionTool[] = []; const queue: RealtimeVoiceToolResult[] = []; let updateSequence = 0; let active: | { result: RealtimeVoiceToolResult; expectedNames: string[]; eventId: string; timer: ReturnType; } | undefined; const finish = (failureMessage?: string) => { if (!active) return; const { result, timer } = active; clearTimeout(timer); active = undefined; send( createRealtimeVoiceToolResultEvent( failureMessage ? { callId: result.callId, status: "failed", output: `${failureMessage} The discovered tools were not added to this voice session. Original tool result: ${result.output}`, } : result, ), ); send({ type: "response.create" }); drain(); }; const drain = () => { if (active) return; const result = queue.shift(); if (!result) return; const expanded = Array.isArray(result.expandedTools) ? result.expandedTools .map(normalizeRealtimeVoiceFunctionTool) .filter((tool): tool is RealtimeVoiceFunctionTool => Boolean(tool)) : []; if (result.status !== "completed" || expanded.length === 0) { send(createRealtimeVoiceToolResultEvent(result)); send({ type: "response.create" }); drain(); return; } const merged = mergeRealtimeVoiceToolManifest(currentTools, expanded); const mergedNames = new Set(merged.map((tool) => tool.name)); const expectedNames = expanded .map((tool) => tool.name) .filter((name) => mergedNames.has(name)); if (expectedNames.length === 0) { send(createRealtimeVoiceToolResultEvent(result)); send({ type: "response.create" }); drain(); return; } const eventId = `realtime_tool_manifest_${++updateSequence}`; active = { result, expectedNames, eventId, timer: setTimeout( () => finish("Timed out while installing the discovered tools."), timeoutMs, ), }; send(createRealtimeVoiceToolManifestUpdate(merged, eventId)); }; return { enqueue(result) { queue.push(result); drain(); }, setSessionTools(tools) { currentTools = mergeRealtimeVoiceToolManifest([], tools); if (!active) return; const names = new Set(currentTools.map((tool) => tool.name)); if (active.expectedNames.every((name) => names.has(name))) finish(); }, handleError(eventId, message) { if (!active || !eventId || active.eventId !== eventId) return false; finish(message || "The provider rejected the discovered tool update."); return true; }, reset() { if (active) clearTimeout(active.timer); active = undefined; queue.length = 0; currentTools = []; }, getTools() { return currentTools; }, }; } export function extractRealtimeVoiceFunctionCalls( event: RealtimeServerEvent, ): Array<{ name: string; callId: string; argumentsText: string }> { if (event.type === "response.function_call_arguments.done") { const name = typeof event.name === "string" ? event.name : ""; const callId = typeof event.call_id === "string" ? event.call_id : ""; if (!name || !callId) return []; return [ { name, callId, argumentsText: typeof event.arguments === "string" ? event.arguments : "{}", }, ]; } if (event.type === "response.output_item.done") { const item = event.item; if (!item || typeof item !== "object") return []; const record = item as Record; if (record.type !== "function_call") return []; const name = typeof record.name === "string" ? record.name : ""; const callId = typeof record.call_id === "string" ? record.call_id : ""; if (!name || !callId) return []; return [ { name, callId, argumentsText: typeof record.arguments === "string" ? record.arguments : "{}", }, ]; } if (event.type !== "response.done") return []; const response = event.response; if (!response || typeof response !== "object") return []; const status = (response as { status?: unknown }).status; if (status !== undefined && status !== "completed") return []; const output = (response as { output?: unknown }).output; if (!Array.isArray(output)) return []; return output.flatMap((item) => { if (!item || typeof item !== "object") return []; const record = item as Record; if (record.type !== "function_call") return []; const name = typeof record.name === "string" ? record.name : ""; const callId = typeof record.call_id === "string" ? record.call_id : ""; if (!name || !callId) return []; return [ { name, callId, argumentsText: typeof record.arguments === "string" ? record.arguments : "{}", }, ]; }); } function parseFunctionArguments(text: string): Record { const parsed = JSON.parse(text || "{}"); if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { throw new Error("The voice model returned invalid tool arguments."); } return parsed as Record; } function sendDataChannelEvent( channel: RTCDataChannel | null, event: Record, ): void { if (!channel || channel.readyState !== "open") return; channel.send(JSON.stringify(event)); } function openOpenAiKeySettings(): void { if (typeof window === "undefined") return; window.location.hash = "#secrets:OPENAI_API_KEY"; window.dispatchEvent(new Event("agent-panel:open")); window.dispatchEvent( new CustomEvent("agent-panel:open-settings", { detail: { section: "secrets" }, }), ); } export function listenForRealtimeVoicePageHide( cleanup: () => void, ): () => void { window.addEventListener("pagehide", cleanup); return () => window.removeEventListener("pagehide", cleanup); } function voiceCopy(t: (key: string) => string): RealtimeVoiceModeCopy { return { entryButtonLabel: t("agentPanel.voiceMode.entryButtonLabel"), promptTitle: t("agentPanel.voiceMode.promptTitle"), promptDescription: t("agentPanel.voiceMode.promptDescription"), setupTitle: t("agentPanel.voiceMode.setupTitle"), setupDescription: t("agentPanel.voiceMode.setupDescription"), connectBuilder: t("agentPanel.voiceMode.connectBuilder"), useOpenAiKey: t("agentPanel.voiceMode.useOpenAiKey"), startWithOpenAiKey: t("agentPanel.voiceMode.startWithOpenAiKey"), startVoiceMode: t("agentPanel.voiceMode.start"), keepDictating: t("agentPanel.voiceMode.keepDictating"), rememberPreference: t("agentPanel.voiceMode.rememberPreference"), showChat: t("agentPanel.voiceMode.showChat"), hideChat: t("agentPanel.voiceMode.hideChat"), endVoiceMode: t("agentPanel.voiceMode.end"), voiceSettings: t("agentPanel.voiceMode.voiceSettings"), settings: { language: t("agentPanel.voiceMode.settings.language"), autoLanguage: t("agentPanel.voiceMode.settings.autoLanguage"), languages: { en: t("agentPanel.voiceMode.settings.languages.en"), es: t("agentPanel.voiceMode.settings.languages.es"), fr: t("agentPanel.voiceMode.settings.languages.fr"), de: t("agentPanel.voiceMode.settings.languages.de"), it: t("agentPanel.voiceMode.settings.languages.it"), pt: t("agentPanel.voiceMode.settings.languages.pt"), ja: t("agentPanel.voiceMode.settings.languages.ja"), ko: t("agentPanel.voiceMode.settings.languages.ko"), zh: t("agentPanel.voiceMode.settings.languages.zh"), }, intelligence: t("agentPanel.voiceMode.settings.intelligence"), intelligenceLevels: { instant: t("agentPanel.voiceMode.settings.intelligenceLevels.instant"), balanced: t( "agentPanel.voiceMode.settings.intelligenceLevels.balanced", ), deep: t("agentPanel.voiceMode.settings.intelligenceLevels.deep"), }, voiceStyle: t("agentPanel.voiceMode.settings.voiceStyle"), microphone: t("agentPanel.voiceMode.settings.microphone"), defaultMicrophone: t("agentPanel.voiceMode.settings.defaultMicrophone"), microphoneSwitchFailed: t( "agentPanel.voiceMode.settings.microphoneSwitchFailed", ), voiceChangePending: t("agentPanel.voiceMode.settings.voiceChangePending"), voiceDescriptions: { marin: t("agentPanel.voiceMode.settings.voiceDescriptions.marin"), cedar: t("agentPanel.voiceMode.settings.voiceDescriptions.cedar"), coral: t("agentPanel.voiceMode.settings.voiceDescriptions.coral"), sage: t("agentPanel.voiceMode.settings.voiceDescriptions.sage"), verse: t("agentPanel.voiceMode.settings.voiceDescriptions.verse"), alloy: t("agentPanel.voiceMode.settings.voiceDescriptions.alloy"), ash: t("agentPanel.voiceMode.settings.voiceDescriptions.ash"), ballad: t("agentPanel.voiceMode.settings.voiceDescriptions.ballad"), echo: t("agentPanel.voiceMode.settings.voiceDescriptions.echo"), shimmer: t("agentPanel.voiceMode.settings.voiceDescriptions.shimmer"), }, }, status: { connecting: t("agentPanel.voiceMode.status.connecting"), listening: t("agentPanel.voiceMode.status.listening"), speaking: t("agentPanel.voiceMode.status.speaking"), working: t("agentPanel.voiceMode.status.working"), error: t("agentPanel.voiceMode.status.error"), ending: t("agentPanel.voiceMode.status.ending"), }, errors: { unsupported: t("agentPanel.voiceMode.errors.unsupported"), responseFailed: t("agentPanel.voiceMode.errors.responseFailed"), sessionFailed: t("agentPanel.voiceMode.errors.sessionFailed"), channelDisconnected: t("agentPanel.voiceMode.errors.channelDisconnected"), connectionTimedOut: t("agentPanel.voiceMode.errors.connectionTimedOut"), connectionFailed: t("agentPanel.voiceMode.errors.connectionFailed"), offerFailed: t("agentPanel.voiceMode.errors.offerFailed"), }, }; } export function useRealtimeVoiceModeCopy(): RealtimeVoiceModeCopy { const { translate: t } = useComposerRuntimeAdapters(); return useMemo(() => voiceCopy(t!), [t]); } function useRealtimeVoiceModeController( browserTabId?: string, copy?: RealtimeVoiceModeCopy, ): RealtimeVoiceModeApi { const adapters = useComposerRuntimeAdapters(); const [state, setState] = useState<"idle" | RealtimeVoiceModeState>("idle"); const [error, setError] = useState(null); const [chatVisible, setChatVisible] = useState(false); const [preferences, setPreferences] = useState( DEFAULT_REALTIME_VOICE_PREFERENCES, ); const [voiceChangePending, setVoiceChangePending] = useState(false); const [microphones, setMicrophones] = useState([]); const [microphoneDeviceId, setMicrophoneDeviceId] = useState( readRealtimeVoiceMicrophoneId, ); const [microphoneSwitching, setMicrophoneSwitching] = useState(false); const [microphoneError, setMicrophoneError] = useState(null); const [audioLevels] = useState(createRealtimeVoiceAudioLevelStore); const stateRef = useRef(state); const preferencesRef = useRef(preferences); const hasOutputAudioRef = useRef(false); const peerRef = useRef(null); const channelRef = useRef(null); const streamRef = useRef(null); const audioRef = useRef(null); const audioContextRef = useRef(null); const inputAnalyserRef = useRef(null); const outputAnalyserRef = useRef(null); const inputMeterBufferRef = useRef | null>(null); const outputMeterBufferRef = useRef | null>(null); const inputSourceRef = useRef(null); const outputSourceRef = useRef(null); const meterFrameRef = useRef(null); const lastMeterSampleRef = useRef(0); const abortRef = useRef(null); const connectionGateRef = useRef | null>(null); const handledCallsRef = useRef(new Set()); const sessionIdRef = useRef(undefined); const capabilityRef = useRef(undefined); const transportGenerationRef = useRef(0); const startedAtRef = useRef(undefined); const lastUserTextRef = useRef(""); const lastAssistantTextRef = useRef(""); const transcriptThreadIdRef = useRef(undefined); const transcriptSequenceRef = useRef(0); const preferencesHydratedRef = useRef(false); const preferencesEditedRef = useRef(false); const preferenceWriteChainRef = useRef>(Promise.resolve()); const responseCoordinator = useMemo( () => createRealtimeVoiceResponseCoordinator((event) => { sendDataChannelEvent(channelRef.current, event); }), [], ); const toolManifestCoordinator = useMemo( () => createRealtimeVoiceToolManifestCoordinator((event) => { if (event.type === "response.create") { responseCoordinator.request(event); } else { sendDataChannelEvent(channelRef.current, event); } }), [responseCoordinator], ); const switchMicrophoneRef = useRef< (deviceId: string, force?: boolean) => Promise >(async () => undefined); useEffect(() => { stateRef.current = state; }, [state]); useEffect(() => { preferencesRef.current = preferences; }, [preferences]); const hydratePreferences = useCallback(async () => { if (preferencesHydratedRef.current) return; try { const stored = await adapters.voice!.readAppState!( REALTIME_VOICE_PREFERENCES_KEY, ); if (!preferencesEditedRef.current && stored != null) { const next = normalizeRealtimeVoicePreferences(stored); preferencesRef.current = next; setPreferences(next); } } catch { // Preferences are optional; keep safe defaults when storage is unavailable. } finally { preferencesHydratedRef.current = true; } }, [adapters]); const syncAppState = useCallback( (nextState: "idle" | RealtimeVoiceModeState) => { const value = nextState === "idle" ? null : { active: true, status: nextState, model: "gpt-realtime-2.1", startedAt: startedAtRef.current, sessionId: sessionIdRef.current, browserTabId, preferences: preferencesRef.current, lastUserText: lastUserTextRef.current || undefined, lastAssistantText: lastAssistantTextRef.current || undefined, }; void Promise.resolve( adapters.voice!.setAppState!(REALTIME_VOICE_STATE_KEY, value), ).catch(() => undefined); }, [adapters, browserTabId], ); const transition = useCallback( (nextState: "idle" | RealtimeVoiceModeState) => { stateRef.current = nextState; setState(nextState); syncAppState(nextState); }, [syncAppState], ); const savePreferences = useCallback( (next: RealtimeVoicePreferences) => { preferencesEditedRef.current = true; preferencesHydratedRef.current = true; preferencesRef.current = next; setPreferences(next); preferenceWriteChainRef.current = preferenceWriteChainRef.current .catch(() => undefined) .then(async () => { await adapters.voice!.setAppState!( REALTIME_VOICE_PREFERENCES_KEY, next, ); }) .catch(() => undefined); }, [adapters], ); const updateLivePreferences = useCallback( (next: RealtimeVoicePreferences, includeVoice = false) => { sendDataChannelEvent( channelRef.current, createRealtimeVoicePreferenceUpdate(next, { browserLanguages: typeof navigator === "undefined" ? [] : navigator.languages, includeVoice, }), ); }, [], ); const setLanguage = useCallback( (language: RealtimeVoiceLanguage) => { const next = { ...preferencesRef.current, language }; savePreferences(next); updateLivePreferences(next); }, [savePreferences, updateLivePreferences], ); const setIntelligence = useCallback( (intelligence: RealtimeVoiceIntelligence) => { const next = { ...preferencesRef.current, intelligence }; savePreferences(next); updateLivePreferences(next); }, [savePreferences, updateLivePreferences], ); const setVoice = useCallback( (voice: RealtimeVoice) => { const next = { ...preferencesRef.current, voice }; savePreferences(next); if (hasOutputAudioRef.current) { setVoiceChangePending(true); updateLivePreferences(next); return; } setVoiceChangePending(false); updateLivePreferences(next, true); }, [savePreferences, updateLivePreferences], ); const refreshMicrophones = useCallback(async (): Promise< RealtimeVoiceMicrophone[] | null > => { const mediaDevices = navigator.mediaDevices; if (!mediaDevices?.enumerateDevices) return null; try { const devices = await mediaDevices.enumerateDevices(); const inputs = devices .filter( (device) => device.kind === "audioinput" && device.deviceId && device.deviceId !== "default" && device.deviceId !== "communications", ) .map((device, index) => ({ deviceId: device.deviceId, label: device.label || `Microphone ${index + 1}`, })); setMicrophones(inputs); return inputs; } catch { // Enumeration is progressive enhancement; system default remains usable. return null; } }, []); const startMeterLoop = useCallback(() => { if (meterFrameRef.current !== null) return; const sample = (timestamp: number) => { meterFrameRef.current = requestAnimationFrame(sample); if (timestamp - lastMeterSampleRef.current < 50) return; lastMeterSampleRef.current = timestamp; const current = audioLevels.getSnapshot(); let input = current.input; let output = current.output; const inputAnalyser = inputAnalyserRef.current; const inputBuffer = inputMeterBufferRef.current; if (inputAnalyser && inputBuffer) { inputAnalyser.getByteTimeDomainData(inputBuffer); input = smoothRealtimeVoiceLevel( input, normalizeRealtimeVoiceRms(inputBuffer), ); } const outputAnalyser = outputAnalyserRef.current; const outputBuffer = outputMeterBufferRef.current; if (outputAnalyser && outputBuffer) { outputAnalyser.getByteTimeDomainData(outputBuffer); output = smoothRealtimeVoiceLevel( output, normalizeRealtimeVoiceRms(outputBuffer), ); } audioLevels.set({ input, output }); }; meterFrameRef.current = requestAnimationFrame(sample); }, [audioLevels]); const attachAudioMeter = useCallback( (stream: MediaStream, channel: "input" | "output") => { try { const AudioCtor = window.AudioContext ?? ( window as typeof window & { webkitAudioContext?: typeof AudioContext; } ).webkitAudioContext; if (!AudioCtor) return; const context = audioContextRef.current ?? new AudioCtor(); audioContextRef.current = context; void context.resume().catch(() => undefined); const analyser = context.createAnalyser(); analyser.fftSize = 512; const source = context.createMediaStreamSource(stream); source.connect(analyser); const buffer = new Uint8Array(analyser.fftSize); if (channel === "input") { inputSourceRef.current?.disconnect(); inputSourceRef.current = source; inputAnalyserRef.current = analyser; inputMeterBufferRef.current = buffer; } else { outputSourceRef.current?.disconnect(); outputSourceRef.current = source; outputAnalyserRef.current = analyser; outputMeterBufferRef.current = buffer; } startMeterLoop(); } catch { // Audio metering is visual-only; keep the realtime call healthy. } }, [startMeterLoop], ); const setMicrophone = useCallback( async (deviceId: string, force = false) => { if ((!force && deviceId === microphoneDeviceId) || microphoneSwitching) { return; } const mediaDevices = navigator.mediaDevices; const peer = peerRef.current; if (!mediaDevices?.getUserMedia || !peer) return; setMicrophoneSwitching(true); setMicrophoneError(null); try { const replacementStream = await replaceRealtimeVoiceMicrophone({ mediaDevices, peer, currentStream: streamRef.current, deviceId, }); const replacementTrack = replacementStream.getAudioTracks()[0]; streamRef.current = replacementStream; if (replacementTrack) { replacementTrack.onended = () => { void switchMicrophoneRef.current("default", true); }; } attachAudioMeter(streamRef.current, "input"); writeRealtimeVoiceMicrophoneId(deviceId); setMicrophoneDeviceId(deviceId); await refreshMicrophones(); } catch { setMicrophoneError( copy?.settings.microphoneSwitchFailed ?? "Could not switch microphones. Your current microphone is still active.", ); } finally { setMicrophoneSwitching(false); } }, [ attachAudioMeter, copy, microphoneDeviceId, microphoneSwitching, refreshMicrophones, ], ); switchMicrophoneRef.current = setMicrophone; const cleanupTransport = useCallback(() => { connectionGateRef.current?.cancel(); connectionGateRef.current = null; transportGenerationRef.current += 1; capabilityRef.current = undefined; abortRef.current?.abort(); abortRef.current = null; channelRef.current?.close(); channelRef.current = null; peerRef.current?.close(); peerRef.current = null; for (const track of streamRef.current?.getTracks() ?? []) { track.onended = null; track.stop(); } streamRef.current = null; if (audioRef.current) { audioRef.current.pause(); audioRef.current.srcObject = null; } audioRef.current = null; if (meterFrameRef.current !== null) { cancelAnimationFrame(meterFrameRef.current); meterFrameRef.current = null; } inputSourceRef.current?.disconnect(); outputSourceRef.current?.disconnect(); inputSourceRef.current = null; outputSourceRef.current = null; inputAnalyserRef.current = null; outputAnalyserRef.current = null; inputMeterBufferRef.current = null; outputMeterBufferRef.current = null; lastMeterSampleRef.current = 0; const audioContext = audioContextRef.current; audioContextRef.current = null; if (audioContext) void audioContext.close().catch(() => undefined); audioLevels.reset(); handledCallsRef.current.clear(); responseCoordinator.reset(); toolManifestCoordinator.reset(); }, [audioLevels, responseCoordinator, toolManifestCoordinator]); const fail = useCallback( (message: string, options?: { openKeySettings?: boolean }) => { cleanupTransport(); setError(message); transition("error"); if (options?.openKeySettings) openOpenAiKeySettings(); }, [cleanupTransport, transition], ); const handleFunctionCall = useCallback( async (call: { name: string; callId: string; argumentsText: string }) => { if (handledCallsRef.current.has(call.callId)) return; handledCallsRef.current.add(call.callId); const transportGeneration = transportGenerationRef.current; const transportChannel = channelRef.current; transition("working"); let result: RealtimeVoiceToolResult; try { const args = parseFunctionArguments(call.argumentsText); result = await executeRealtimeVoiceTool({ name: call.name, args, callId: call.callId, sessionId: sessionIdRef.current, browserTabId, capability: capabilityRef.current, signal: abortRef.current?.signal, }); } catch (toolError) { result = { callId: call.callId, status: "failed", output: errorMessage(toolError), }; } if ( transportGeneration !== transportGenerationRef.current || channelRef.current !== transportChannel ) { return; } toolManifestCoordinator.enqueue(result); }, [browserTabId, toolManifestCoordinator, transition], ); const publishCompletedTranscript = useCallback( (transcript: CompletedRealtimeVoiceTranscript) => { const threadId = transcriptThreadIdRef.current; if (!threadId) return; const sessionIdentity = sessionIdRef.current ?? startedAtRef.current ?? "pending"; const providerIdentity = transcript.providerId ?? `sequence-${++transcriptSequenceRef.current}`; realtimeVoiceTranscriptRegistry.publish({ id: `realtime-voice:${sessionIdentity}:${transcript.role}:${providerIdentity}`, threadId, role: transcript.role, text: transcript.text, createdAt: new Date().toISOString(), }); }, [], ); const transcriptSequencer = useMemo( () => createRealtimeVoiceTranscriptSequencer(publishCompletedTranscript), [publishCompletedTranscript], ); const greetingStarter = useMemo( () => createRealtimeVoiceGreetingStarter((event) => { responseCoordinator.request(event); }), [responseCoordinator], ); const handleServerEvent = useCallback( (event: RealtimeServerEvent) => { responseCoordinator.handleEvent(event); transcriptSequencer.handle(event); const sessionTools = extractRealtimeVoiceSessionTools(event); if (sessionTools) toolManifestCoordinator.setSessionTools(sessionTools); if (event.type === "session.created") { connectionGateRef.current?.markSessionCreated(); connectionGateRef.current = null; const session = event.session; if (session && typeof session === "object") { const id = (session as { id?: unknown }).id; if (typeof id === "string") sessionIdRef.current = id; } updateLivePreferences(preferencesRef.current, true); greetingStarter.start(); transition("working"); } else if (event.type === "input_audio_buffer.speech_started") { transition("listening"); } else if (event.type === "input_audio_buffer.speech_stopped") { transition("working"); responseCoordinator.request(); } else if (event.type === "response.created") { // From this point onward the response is committed to audio output, so // changing voices risks violating Realtime's per-session voice lock. hasOutputAudioRef.current = true; lastAssistantTextRef.current = ""; transition("working"); } else if (event.type === "response.output_audio_transcript.delta") { hasOutputAudioRef.current = true; if (typeof event.delta === "string") { lastAssistantTextRef.current += event.delta; } transition("speaking"); } else if (event.type === "response.output_audio_transcript.done") { if (typeof event.transcript === "string") { lastAssistantTextRef.current = event.transcript; } syncAppState("speaking"); } else if ( event.type === "conversation.item.input_audio_transcription.completed" ) { if (typeof event.transcript === "string") { lastUserTextRef.current = event.transcript; } syncAppState("working"); } else if (event.type === "response.done") { const response = event.response; const status = response && typeof response === "object" ? (response as { status?: unknown }).status : undefined; if (status === "failed") { transportGenerationRef.current += 1; handledCallsRef.current.clear(); responseCoordinator.reset(); toolManifestCoordinator.reset(); transition("listening"); return; } } else if (event.type === "response.failed") { transportGenerationRef.current += 1; handledCallsRef.current.clear(); responseCoordinator.reset(); toolManifestCoordinator.reset(); transition("listening"); return; } else if (event.type === "error") { const detail = event.error; const message = detail && typeof detail === "object" ? String((detail as { message?: unknown }).message ?? "") : typeof detail === "string" ? detail : ""; const expansionEventId = detail && typeof detail === "object" ? ((detail as { event_id?: unknown }).event_id ?? event.event_id) : event.event_id; if ( toolManifestCoordinator.handleError( typeof expansionEventId === "string" ? expansionEventId : undefined, message, ) ) { return; } if (responseCoordinator.handleError(message)) { transition("working"); return; } fail( message || copy?.errors.sessionFailed || "The realtime voice session encountered an error.", ); return; } const calls = extractRealtimeVoiceFunctionCalls(event); for (const call of calls) void handleFunctionCall(call); if (event.type === "response.done" && calls.length === 0) { transition("listening"); } }, [ copy, fail, greetingStarter, handleFunctionCall, syncAppState, transcriptSequencer, transition, responseCoordinator, toolManifestCoordinator, updateLivePreferences, ], ); const start = useCallback(async () => { if (stateRef.current !== "idle") return; if ( typeof RTCPeerConnection === "undefined" || !navigator.mediaDevices?.getUserMedia ) { fail( copy?.errors.unsupported ?? "This browser does not support realtime voice conversations.", ); return; } setError(null); startedAtRef.current = new Date().toISOString(); lastUserTextRef.current = ""; lastAssistantTextRef.current = ""; sessionIdRef.current = undefined; hasOutputAudioRef.current = false; greetingStarter.reset(); setVoiceChangePending(false); transcriptThreadIdRef.current = realtimeVoiceTranscriptRegistry.activeThreadId(); transcriptSequenceRef.current = 0; transcriptSequencer.reset(); transition("connecting"); setChatVisible(false); window.dispatchEvent(new Event("agent-panel:close")); await hydratePreferences(); if ((stateRef.current as string) !== "connecting") return; const abortController = new AbortController(); abortRef.current = abortController; const isCurrentAttempt = () => abortRef.current === abortController && !abortController.signal.aborted; const connectionGate = createRealtimeVoiceConnectionGate(() => { if (!isCurrentAttempt()) return; fail( copy?.errors.connectionTimedOut ?? "The realtime voice connection timed out.", ); }); connectionGateRef.current = connectionGate; try { const stream = await navigator.mediaDevices.getUserMedia({ audio: createRealtimeVoiceAudioConstraints( readRealtimeVoiceMicrophoneId(), ), }); if (!isCurrentAttempt()) { for (const track of stream.getTracks()) track.stop(); return; } streamRef.current = stream; for (const track of stream.getAudioTracks()) { track.onended = () => { void switchMicrophoneRef.current("default", true); }; } attachAudioMeter(stream, "input"); void refreshMicrophones().then((inputs) => { const selected = readRealtimeVoiceMicrophoneId(); if ( inputs && selected !== "default" && !inputs.some((device) => device.deviceId === selected) ) { // The persisted device was only an ideal preference, so getUserMedia // already selected a usable fallback for this new session. writeRealtimeVoiceMicrophoneId("default"); setMicrophoneDeviceId("default"); } }); const peer = new RTCPeerConnection(); peerRef.current = peer; for (const track of stream.getAudioTracks()) peer.addTrack(track, stream); const audio = document.createElement("audio"); audio.autoplay = true; audio.setAttribute("playsinline", ""); audioRef.current = audio; peer.ontrack = (trackEvent) => { const remoteStream = trackEvent.streams[0] ?? new MediaStream([trackEvent.track]); audio.srcObject = remoteStream; attachAudioMeter(remoteStream, "output"); void audio.play().catch(() => undefined); }; const channel = peer.createDataChannel("oai-events"); channelRef.current = channel; channel.onopen = connectionGate.markTransportReady; channel.onmessage = (messageEvent) => { if (!isCurrentAttempt()) return; try { handleServerEvent(JSON.parse(String(messageEvent.data))); } catch { // Ignore malformed provider events without ending a healthy call. } }; channel.onerror = () => { if (!isCurrentAttempt()) return; fail( copy?.errors.channelDisconnected ?? "The realtime voice control channel disconnected.", ); }; peer.onconnectionstatechange = () => { if (!isCurrentAttempt()) return; if (peer.connectionState === "connected") { connectionGate.markTransportReady(); } if (peer.connectionState === "failed") { fail( copy?.errors.connectionFailed ?? "The realtime voice connection failed.", ); } }; const offer = await peer.createOffer(); if (!isCurrentAttempt()) return; await peer.setLocalDescription(offer); if (!isCurrentAttempt()) return; if (!offer.sdp) { throw new Error( copy?.errors.offerFailed ?? "The browser did not create an audio offer.", ); } const answer = await createRealtimeVoiceSessionWithCapability(offer.sdp, { browserTabId, signal: abortController.signal, preferences: preferencesRef.current, browserLanguages: navigator.languages, }); if (!isCurrentAttempt()) return; capabilityRef.current = answer.capability; await peer.setRemoteDescription({ type: "answer", sdp: answer.sdp }); } catch (startError) { // Ending a connection aborts the SDP request by design. A superseded // attempt must never resurrect the dock or surface that cancellation as // an error after cleanup. if ( isRealtimeVoiceAbortError(startError) || abortController.signal.aborted || abortRef.current !== abortController ) { return; } if (isRealtimeVoiceSetupRequiredError(startError)) { // Credentials can disappear after the entry-point status check. Treat // the authoritative setup response as a gate, not as an active voice // session failure: clean up the mic/RTC attempt, reopen chat, and // refresh both the chat and voice provider setup surfaces. cleanupTransport(); setError(null); startedAtRef.current = undefined; sessionIdRef.current = undefined; transcriptThreadIdRef.current = undefined; transition("idle"); setChatVisible(true); window.dispatchEvent(new Event("agent-panel:open")); window.dispatchEvent(new Event("agent-engine:configured-changed")); return; } const status = (startError as { status?: unknown })?.status; fail(errorMessage(startError), { openKeySettings: status === 400 }); } }, [ attachAudioMeter, browserTabId, cleanupTransport, copy, fail, greetingStarter, handleServerEvent, hydratePreferences, refreshMicrophones, transcriptSequencer, transition, ]); const end = useCallback(() => { if (stateRef.current === "idle" || stateRef.current === "ending") return; const transcriptThreadId = transcriptThreadIdRef.current; const activeThreadId = realtimeVoiceTranscriptRegistry.activeThreadId(); transition("ending"); cleanupTransport(); setError(null); sessionIdRef.current = undefined; startedAtRef.current = undefined; transcriptThreadIdRef.current = undefined; setChatVisible(true); if ( shouldRestoreRealtimeVoiceTranscriptThread( transcriptThreadId, activeThreadId, ) ) { adapters.agentChat!.requestThreadOpen!({ threadId: transcriptThreadId, // The request is delivered asynchronously. Re-checking this at the // receiver prevents a navigation that happened during that gap from // being overwritten. onlyIfActiveThreadId: transcriptThreadId, }); } else { window.dispatchEvent(new Event("agent-panel:open")); } transition("idle"); }, [adapters, cleanupTransport, transition]); const toggleChat = useCallback(() => { setChatVisible((current) => !current); window.dispatchEvent(new Event("agent-panel:toggle")); }, []); useEffect(() => { const mediaDevices = navigator.mediaDevices; if (!mediaDevices?.addEventListener) return; const handleDeviceChange = () => { const selected = readRealtimeVoiceMicrophoneId(); void refreshMicrophones().then((inputs) => { if ( inputs && selected !== "default" && !inputs.some((device) => device.deviceId === selected) ) { void setMicrophone("default"); } }); }; mediaDevices.addEventListener("devicechange", handleDeviceChange); return () => mediaDevices.removeEventListener("devicechange", handleDeviceChange); }, [refreshMicrophones, setMicrophone]); useEffect(() => { return adapters.voice!.subscribeSidebarState!((detail) => { if (detail && typeof detail.open === "boolean") { setChatVisible(detail.open); } }); }, [adapters]); useEffect(() => { const stopListening = listenForRealtimeVoicePageHide(cleanupTransport); return () => { stopListening(); cleanupTransport(); }; }, [cleanupTransport]); return { state, active: state !== "idle", errorMessage: error, chatVisible, audioLevels, preferences, microphones, microphoneDeviceId, microphoneSwitching, microphoneError, voiceChangePending, setLanguage, setIntelligence, setVoice, setMicrophone, start, end, toggleChat, }; } const RealtimeVoiceModeContext = createContext( null, ); export function RealtimeVoiceModeProvider({ children, browserTabId, }: RealtimeVoiceModeProviderProps) { const adapters = useComposerRuntimeAdapters(); const resolvedBrowserTabId = useMemo( () => browserTabId ?? adapters.voice!.getBrowserTabId!(), [adapters, browserTabId], ); const copy = useRealtimeVoiceModeCopy(); const voice = useRealtimeVoiceModeController(resolvedBrowserTabId, copy); const inlineSettings = useMemo( () => ({ dialogLabel: copy.voiceSettings, ...(voice.voiceChangePending ? { appliesNextConversationNote: copy.settings.voiceChangePending } : {}), ...(voice.microphoneError ? { microphoneError: voice.microphoneError } : {}), microphone: { label: copy.settings.microphone, value: voice.microphoneDeviceId, disabled: voice.microphoneSwitching, options: [ { value: "default", label: copy.settings.defaultMicrophone, }, ...voice.microphones.map((microphone) => ({ value: microphone.deviceId, label: microphone.label, })), ], onValueChange: (value: string) => { void voice.setMicrophone(value); }, }, language: { label: copy.settings.language, value: voice.preferences.language, options: REALTIME_VOICE_LANGUAGES.map((language) => ({ value: language, label: language === "auto" ? copy.settings.autoLanguage : copy.settings.languages[language], })), onValueChange: (value: string) => { if (isOneOf(REALTIME_VOICE_LANGUAGES, value)) { voice.setLanguage(value); } }, }, intelligence: { label: copy.settings.intelligence, value: voice.preferences.intelligence, options: REALTIME_VOICE_INTELLIGENCE_LEVELS.map((intelligence) => ({ value: intelligence, label: copy.settings.intelligenceLevels[intelligence], })), onValueChange: (value: string) => { if (isOneOf(REALTIME_VOICE_INTELLIGENCE_LEVELS, value)) { voice.setIntelligence(value); } }, }, voiceStyle: { label: copy.settings.voiceStyle, value: voice.preferences.voice, options: REALTIME_VOICES.map((voiceName) => ({ value: voiceName, label: `${voiceName[0]!.toUpperCase()}${voiceName.slice(1)}`, description: copy.settings.voiceDescriptions[voiceName], })), onValueChange: (value: string) => { if (isOneOf(REALTIME_VOICES, value)) voice.setVoice(value); }, }, }), [copy, voice], ); return ( {children} {voice.active && typeof document !== "undefined" ? createPortal( , document.body, ) : null} ); } /** * Ensure standalone/full-page composers get realtime voice without nesting a * second session owner inside the persistent AgentSidebar provider. */ export function RealtimeVoiceModeBoundary({ children, browserTabId, }: RealtimeVoiceModeProviderProps) { const existing = useRealtimeVoiceModeOptional(); if (existing) { return ( {children} ); } return ( {children} ); } /** Hide a composer while voice owns input; the dock can reveal it on demand. */ function RealtimeVoiceModeComposerSurface({ children, }: Pick) { const voice = useRealtimeVoiceModeOptional(); if (voice?.active && !voice.chatVisible) return null; return children; } export function useRealtimeVoiceMode(): RealtimeVoiceModeApi { const value = useContext(RealtimeVoiceModeContext); if (!value) { throw new Error( "useRealtimeVoiceMode must be used inside RealtimeVoiceModeProvider.", ); } return value; } export function useRealtimeVoiceModeOptional(): RealtimeVoiceModeApi | null { return useContext(RealtimeVoiceModeContext); } export async function readRealtimeVoiceContext(): Promise<{ navigation: unknown; url: unknown; }> { return readRealtimeVoiceContextWith({}); } export async function readRealtimeVoiceContextWith(options: { readAppState?: (key: string) => Promise | unknown; resolvePath?: (path: string) => string; }): Promise<{ navigation: unknown; url: unknown }> { if (options.readAppState) { const [navigation, url] = await Promise.all([ Promise.resolve(options.readAppState("navigation")).catch(() => null), Promise.resolve(options.readAppState("__url__")).catch(() => null), ]); return { navigation, url }; } const resolvePath = options.resolvePath ?? ((path: string) => path); const read = async (key: string) => { const response = await fetch( resolvePath( `/_agent-native/application-state/${encodeURIComponent(key)}`, ), ); if (!response.ok) return null; const body = (await response.json()) as { value?: unknown } | unknown; return body && typeof body === "object" && "value" in body ? ((body as { value?: unknown }).value ?? null) : body; }; const [navigation, url] = await Promise.all([ read("navigation").catch(() => null), read("__url__").catch(() => null), ]); return { navigation, url }; }