/** * RealtimeAgent - Real-time voice/audio interaction agent * * Python parity with praisonaiagents/agent/realtime_agent.py * Enables real-time voice conversations over a real WebSocket connection. * * ## Why the WebSocket is loaded the way it is * * This module is exported from the package root, so it has to stay loadable in * a browser/webview bundle (see `scripts/webview-gate.mjs`). A static * `import WebSocket from 'ws'` would put an unresolvable bare specifier on that * bundle. So the implementation is resolved at call time: * * 1. an explicitly injected constructor (`{ webSocket }`), for tests and for * runtimes that ship their own, * 2. `globalThis.WebSocket` -- browsers, React Native, and Node >= 22, * 3. the optional `ws` package, imported through a computed specifier so no * bundler tries to resolve it statically. `ws` is NOT a dependency of this * package; this branch only fires if a consumer installed it themselves, * 4. otherwise a clear error naming all three ways to fix it. It never * pretends to have connected. */ /** * The subset of the WebSocket API this agent uses. * * Both the WHATWG `WebSocket` (browsers, Node >= 22) and the `ws` package * satisfy this, which is why a single code path can drive either. */ export interface WebSocketLike { readyState?: number; send(data: string | ArrayBufferLike | ArrayBufferView): void; close(code?: number, reason?: string): void; addEventListener(type: string, listener: (event: any) => void): void; removeEventListener?(type: string, listener: (event: any) => void): void; } /** Constructor shape for a {@link WebSocketLike}. */ export type WebSocketConstructorLike = new (url: string, protocolsOrOptions?: any, options?: any) => WebSocketLike; interface ResolvedWebSocket { ctor: WebSocketConstructorLike; /** Can this implementation send custom HTTP headers on the handshake? */ supportsHeaders: boolean; /** Human-readable origin, used in error messages only. */ source: string; } /** Reset the cached WebSocket resolution. Exported for tests. */ export declare function _resetWebSocketResolution(): void; /** * Resolve a WebSocket implementation, preferring a global one. * * @throws if no implementation can be found - never silently degrades. */ export declare function resolveWebSocketImplementation(injected?: WebSocketConstructorLike, /** * Loader for the optional `ws` package. Defaults to a dynamic import through * a computed specifier. Overridable in tests so the "no implementation * available" path can be exercised without uninstalling a transitive `ws`. */ wsLoader?: () => Promise): Promise; /** * Configuration for Realtime settings. */ export interface RealtimeConfig { /** Voice to use for responses */ voice?: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer'; /** Audio input format */ inputFormat?: 'pcm16' | 'g711_ulaw' | 'g711_alaw'; /** Audio output format */ outputFormat?: 'pcm16' | 'g711_ulaw' | 'g711_alaw'; /** Sample rate in Hz (local playback hint; not sent to the API) */ sampleRate?: number; /** Enable voice activity detection */ vadEnabled?: boolean; /** VAD threshold */ vadThreshold?: number; /** Timeout in seconds */ timeout?: number; } /** * Event types the agent knows by name. The server may send others; they are * emitted verbatim rather than dropped, which is why {@link RealtimeEventType} * also admits arbitrary strings. */ export type KnownRealtimeEventType = 'session.created' | 'session.updated' | 'input_audio_buffer.append' | 'input_audio_buffer.commit' | 'input_audio_buffer.clear' | 'response.create' | 'response.done' | 'response.audio.delta' | 'response.audio.done' | 'response.text.delta' | 'response.text.done' | 'error'; /** * Event type for realtime sessions. * * `'*'` is a local wildcard: a handler registered for it receives every event. * Any other string is a server event type, passed through untouched. */ export type RealtimeEventType = KnownRealtimeEventType | '*' | (string & {}); /** * Realtime event. */ export interface RealtimeEvent { type: RealtimeEventType; /** The full decoded server payload for this event. */ data?: any; timestamp?: number; } /** * Configuration for creating a RealtimeAgent. */ export interface RealtimeAgentConfig { /** Agent name */ name?: string; /** Model to use */ llm?: string; /** Realtime configuration */ realtime?: boolean | RealtimeConfig; /** System instructions */ instructions?: string; /** Enable verbose output */ verbose?: boolean; /** API key */ apiKey?: string; /** * Full WebSocket endpoint override (e.g. a local test server or a proxy). * When set, the `model` query parameter is not appended and a missing API * key is not an error. */ url?: string; /** Extra handshake headers (ignored by implementations that cannot send them) */ headers?: Record; /** Inject a WebSocket constructor instead of resolving one */ webSocket?: WebSocketConstructorLike; /** How long to wait for the handshake before failing. Default 30000ms. */ connectTimeoutMs?: number; } /** * Agent for real-time voice conversations. * * Uses a real WebSocket connection for low-latency audio streaming. * `connect()` resolves only once the handshake has actually completed; a bad * key, a bad URL or a refused connection rejects, and `isConnected()` stays * false. * * @example * ```typescript * import { RealtimeAgent } from 'praisonai'; * * const agent = new RealtimeAgent({ * instructions: 'You are a helpful voice assistant', * realtime: { voice: 'nova' } * }); * * // Listen for what the server actually sends * agent.on('response.audio.delta', (event) => { * playAudio(event.data); * }); * * await agent.connect(); // rejects if the handshake fails * agent.sendAudio(audioBuffer); * await agent.disconnect(); * ``` */ export declare class RealtimeAgent { static readonly DEFAULT_MODEL = "gpt-4o-realtime-preview"; readonly name: string; readonly llm: string; private readonly instructions?; private readonly verbose; private readonly realtimeConfig; private readonly apiKey?; private readonly urlOverride?; private readonly extraHeaders; private readonly webSocketImpl?; private readonly connectTimeoutMs; private connected; private ws; /** In-flight connect(), so concurrent callers share one handshake. */ private connecting; /** The socket being opened by the in-flight connect(), for cancellation. */ private connectingSocket; /** Bumped on every teardown/disconnect so a stale handshake can abort. */ private connectGeneration; private eventHandlers; private messageCallbacks; private audioCallbacks; private errorCallbacks; constructor(config: RealtimeAgentConfig); private log; /** The endpoint this agent will dial. Never contains the API key. */ getUrl(): string; /** The `session.update` payload derived from the local config. */ private sessionPayload; /** * Connect to the realtime session. * * Resolves only after the WebSocket handshake has completed. Rejects on a * refused connection, a non-101 response (which is what a bad API key looks * like), a close during the handshake, or a timeout. */ connect(): Promise; private doConnect; /** * Disconnect from the realtime session. */ disconnect(): Promise; /** * Drop the socket reference and mark the agent disconnected. * * Bumps {@link connectGeneration} so any handshake still in flight is * invalidated: it will close its socket and reject rather than flip the * agent back to connected. */ private teardown; /** * Check if connected. Reflects the live socket state, not an intention. */ isConnected(): boolean; /** * Send a raw event over the socket. * * @throws if not connected - it never silently drops the event. */ sendEvent(event: Record): Promise; /** * Send audio data to the session (base64-encoded, per the Realtime API). */ sendAudio(audioData: ArrayBuffer | Uint8Array): void; /** * Commit the audio buffer for processing. */ commitAudio(): void; /** * Clear the audio buffer. */ clearAudio(): void; /** * Send a text message and request a response (Python `asend_text` parity). */ sendText(text: string): Promise; private handleMessage; private handleClose; private reportError; /** * Invoke user callbacks over a snapshot of the list, isolating each one so a * throwing handler cannot skip later handlers or stop the frame from reaching * `emit`. Mirrors the isolation already applied to error callbacks. */ private invokeCallbacks; /** * Register an event handler. Use `'*'` to receive every server event. */ on(eventType: RealtimeEventType, handler: (event: RealtimeEvent) => void): void; /** * Remove an event handler. */ off(eventType: RealtimeEventType, handler: (event: RealtimeEvent) => void): void; /** Register a callback for text deltas (Python `on_message` parity). */ onMessage(callback: (text: string) => void): void; /** Register a callback for decoded audio deltas (Python `on_audio` parity). */ onAudio(callback: (audio: Uint8Array) => void): void; /** Register a callback for errors (Python `on_error` parity). */ onError(callback: (error: Error) => void): void; /** * Emit an event to handlers. */ private emit; /** * Get the current configuration. */ getConfig(): Required; /** * Update the session configuration. Pushes `session.update` when connected. */ updateConfig(config: Partial): Promise; } /** * Create a RealtimeAgent instance. */ export declare function createRealtimeAgent(config: RealtimeAgentConfig): RealtimeAgent; export {};