import * as dntShim from "../_dnt.shims.js"; import EventEmitter from "./event_emitter.js"; import type * as Events from "./events.js"; import type * as Types from "./types.js"; import RTCMediaManager from "./RTCMediaManager.js"; /** * Typed event emitter to enable type safety for Realtime API events. */ declare class TypedEventEmitter> { emitter: EventEmitter; /** * Listen for server-sent events. Full listing of server-sent events can be * found here: * * https://platform.openai.com/docs/api-reference/realtime-server-events * * @param eventName the name of the event to listen for * @param handler a function to handle the event data */ on(eventName: TEventName, handler: (...eventArg: TEvents[TEventName]) => void): void; /** * Listen for a server-sent event once. Full listing of server-sent events can * be found here: * * https://platform.openai.com/docs/api-reference/realtime-server-events * * @param eventName the name of the event to listen for * @param handler a function to handle the event data */ once(eventName: TEventName, handler: (...eventArg: TEvents[TEventName]) => void): void; /** * Remove an event handler for a particular event. * * @param eventName the name of the event you wish to deregister * @param handler the handler to deregister */ off(eventName: TEventName, handler: (...eventArg: TEvents[TEventName]) => void): void; } /** * Union type for possible client events */ type ClientEvents = Events.SessionUpdateEvent | Events.InputAudioBufferAppendEvent | Events.InputAudioBufferCommitEvent | Events.InputAudioBufferClearEvent | Events.ConversationItemCreateEvent | Events.ConversationItemTruncateEvent | Events.ConversationItemDeleteEvent | Events.ResponseCreateEvent | Events.ResponseCancelEvent | string; /** * Map server event names to event data shapes. */ type ServerEvents = { "error": [Events.ErrorEvent]; "session.created": [Events.SessionCreatedEvent]; "session.updated": [Events.SessionUpdatedEvent]; "conversation.created": [Events.ConversationCreatedEvent]; "conversation.item.created": [Events.ConversationItemCreatedEvent]; "conversation.item.truncated": [Events.ConversationItemTruncatedEvent]; "conversation.item.deleted": [Events.ConversationItemDeletedEvent]; "conversation.item.input_audio_transcription.completed": [ Events.ConversationItemInputAudioTranscriptionCompletedEvent ]; "conversation.item.input_audio_transcription.failed": [ Events.ConversationItemInputAudioTranscriptionFailedEvent ]; "input_audio_buffer.committed": [Events.InputAudioBufferCommittedEvent]; "input_audio_buffer.cleared": [Events.InputAudioBufferClearedEvent]; "input_audio_buffer.speech_started": [ Events.InputAudioBufferSpeechStartedEvent ]; "input_audio_buffer.speech_stopped": [ Events.InputAudioBufferSpeechStoppedEvent ]; "response.created": [Events.ResponseCreatedEvent]; "response.done": [Events.ResponseDoneEvent]; "response.output_item.added": [Events.ResponseOutputItemAddedEvent]; "response.output_item.done": [Events.ResponseOutputItemDoneEvent]; "response.content_part.added": [Events.ResponseContentPartItemAddedEvent]; "response.content_part.done": [Events.ResponseContentPartItemDoneEvent]; "response.text.delta": [Events.ResponseTextDeltaEvent]; "response.text.done": [Events.ResponseTextDoneEvent]; "response.audio_transcript.delta": [Events.ResponseAudioTranscriptDeltaEvent]; "response.audio_transcript.done": [Events.ResponseAudioTranscriptDoneEvent]; "response.audio.delta": [Events.ResponseAudioDeltaEvent]; "response.audio.done": [Events.ResponseAudioDoneEvent]; "response.function_call_arguments.delta": [ Events.ResponseFunctionCallArgumentsDeltaEvent ]; "response.function_call_arguments.done": [ Events.ResponseFunctionCallArgumentsDoneEvent ]; "rate_limits.updated": [Events.RateLimitsUpdatedEvent]; }; /** * Constructor arguments for the Realtime client. */ type OpenAIRealtimeClientOptions = { /** * Enable debug logging */ debug?: boolean; /** * An API key to use to connect to the Realtime API. For server-to-server * communication, a standard OpenAI API key is fine. For browsers that want * to connect to the Realtime API, this should be an ephemeral key. */ apiKey?: string; /** * Provide a URL where an ephemeral API key can be retrieved from via HTTP. * Uses a GET request by default. The key will be retrieved from this URL * every time a new session is started. */ apiKeyUrl?: string; /** * Initialization options for the fetch request used to obtain an * ephemeral key - used in conjunction with `apiKeyUrl`. */ apiKeyFetchOptions?: RequestInit; /** * Optionally override the API endpoint used for Realtime. */ baseUrl?: string; /** * Provide a custom Realtime model ID to use. */ model?: Types.RealtimeModelId; /** * The Realtime model voice to use. */ voice?: Types.Voice; /** * Default instructions for the Realtime model. */ instructions?: string; /** * Use a WebSocket even if RTCPeerConnection is available. Default false. */ useWebSocket?: boolean; }; /** * Initialization options for the Realtime API client */ type OpenAIRealtimeClientStartOptions = { /** * Override the default configuration logic of the RTCPeerConnection with * your own logic. The default logic for handling user media and audio is * pretty naive - use this hook to configure more advanced media * controls for your use case. See also: * * https://webrtc.org/getting-started/media-devices * * This function is called before the session is initialized and the SDP * offer/answer is initiated. * * @param connection The RTCPeerConnection object to configure */ configurePeerConnection?: (connection: RTCPeerConnection) => Promise; /** * Add additional configuration to the RTCDataChannel that the client will use * to send and receive messages from the server. This data channel powers the * client's typed EventEmitter. * * This function is called before the session is initialized and the SDP * offer/answer is initiated. * * @param dataChannel The RTCDataChannel object to configure */ configureDataChannel?: (dataChannel: RTCDataChannel) => void; /** * Options for the WebRTC RTCPeerConnection constructor. See also: * * https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/RTCPeerConnection */ rtcConfiguration?: RTCConfiguration; /** * Optional configuration for the SDP offer. See also: * * https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/createOffer */ offerOptions?: RTCOfferOptions; }; /** * The OpenAIRealtimeClient enables communication with Realtime models over * either WebRTC (default in environments that support it) or over a WebSocket. */ export default class OpenAIRealtimeClient extends TypedEventEmitter { /** * Protocol to use for connection - can be https:// or wss:// */ connectionProtocol: "https://" | "wss://"; /** * Endpoint used to establish a connection to the Realtime API. */ baseUrl: string; /** * Realtime model ID you wish to use. Defaults to latest GPT-4o that * supports Realtime. */ model: Types.RealtimeModelId; /** * Default instructions for the Realtime model. */ instructions: string; /** * The voice the realtime model should use. Defaults to "sage". */ voice: Types.Voice; /** * A WebSocket used in Node.js environments (backed by ws) when * RTCPeerConnection is unavailable in the global scope. */ socket?: dntShim.WebSocket; /** * A WebRTC peer connection, intiialized using a valid OpenAI API token. Null * prior to a successful "init" call. In Node.js, will be null, as a WebSocket * will be used instead. */ peerConnection?: RTCPeerConnection; /** * WebRTC data channel used to send and receive events from the Realtime API. */ dataChannel?: RTCDataChannel; /** * API key used to authenticate a WebRTC session. */ apiKey?: string; /** * URL used to retrieve an ephemeral client key on every `start` request. */ apiKeyUrl?: string; /** * Initialization options for the fetch request used to obtain an * ephemeral key - used in conjunction with `apiKeyUrl`. */ apiKeyFetchOptions?: RequestInit; /** * Whether to enable debug logging from the client. */ debug: boolean; /** * Whether to use a WebSocket even if RTCPeerConnection is available in the * global scope. */ useWebSocket: boolean; /** * A basic default set of logic for handling audio input devices, and playing * remote media tracks sent from the Realtime model. Will be null if * `configurePeerConnection` option is used when starting the session, as * that option will give the developer full control over media streams and * user input device management. */ mediaManager?: RTCMediaManager; /** * Create a new Realtime API Client. * * @param options optional configuration for the client */ constructor(options?: OpenAIRealtimeClientOptions); /** * Start a Realtime API session. If an API key was provided when you created * the client, that API key is used to connect. If you provided an apiKeyUrl, * that URL is fetched to return an ephemeral key. * * The fetch assumes that the URL will return a Session resource like the one * returned in the REST API from a POST to /v1/realtime/sessions. This * needs to happen on the server, so you can safely use a standard API key. * Your endpoint should return JSON that has, at minimum: * * { * client_secret: { value: "your_ephemeral_key" } * } * * If you just return the JSON you get from /v1/realtime/sessions, this will * already be true. * * The optional options argument allows you to configure the behavior of * client initialization as necessary. * * See https://platform.openai.com/docs/guides/realtime for more information * and examples. * * @param keyOrUrl - an ephemeral key, or a URL where one can be created * @param options - optional parameters for initialization */ start(options?: OpenAIRealtimeClientStartOptions): Promise; /** * Disconnect from API server. */ stop(): void; /** * Initialize a WebRTC connection and start a session. * * @param options session initialization options */ private configureWebRTC; /** * Set up basic audio handling with a managed