import { EventEmitter } from './eventEmitter.js'; import { UserState, AgentState, ChatRole } from './enums.js'; import { ChatContext, ChatMessage, ImageContent } from './chat.js'; import type { AgentHandoff } from './chat.js'; import { UtteranceHandle } from './utterance.js'; import type { Agent } from './agent.js'; import type { Pipeline } from './pipeline.js'; /** * Names of the events an {@link AgentSession} emits during a call - state changes, * turn boundaries, transcripts, synthesis lifecycle, DTMF, A2A/pub-sub, recording, * and more. Subscribe with `session.on(event, handler)`. */ export type AgentSessionEvent = 'user_state_changed' | 'agent_state_changed' | 'recording_status_changed' | 'participant_joined' | 'participant_left' | 'runtime_warning' | 'runtime_error' | 'interrupt' | 'metrics_collected' | 'dtmf_received' | 'voicemail_detected' | 'a2a_message' | 'pubsub_message' | 'user_turn_start' | 'user_turn_end' | 'agent_turn_start' | 'agent_turn_end' | 'session_ended' | 'stream_enabled' | 'stream_disabled' | 'transcript_preflight' | 'eou_detected' | 'generation_chunk' | 'vad_event' | 'generation_started' | 'generation_complete' | 'synthesis_started' | 'first_audio_byte' | 'last_audio_byte' | 'synthesis_interrupted' | 'word_timing' | 'tts_capabilities' | 'stt_stream_started' | 'stt_stream_ended' | 'wake_up_exceeded' | 'meeting_joined' | 'meeting_left' | 'agent_joined' | 'agent_left' | 'speech_in' | 'component_metrics' | 'turn_metrics'; type AgentSessionEvents = Record & Record; /** Options for constructing an {@link AgentSession}. */ export type AgentSessionOptions = { /** * Inactivity timeout in seconds after which the agent proactively wakes the user * (requires an `onWakeUp` handler), or `null` to disable. Defaults to `null`. */ wakeUp?: number | null; /** Maximum number of wake-up attempts before ending the call, or `null` for unlimited. Defaults to `null`. */ wakeUpMaxAttempts?: number | null; /** Background audio to play during the call (config object, URL, or path). Defaults to `null`. */ backgroundAudio?: any; /** Handler for incoming DTMF (touch-tone) digits. Defaults to the pipeline's, else `null`. */ dtmfHandler?: any; /** Voicemail detector. Defaults to the pipeline's, else `null`. */ voiceMailDetector?: any; /** Recording configuration applied to the session. Defaults to `null`. */ recording?: any; }; /** Options for {@link AgentSession.start}. */ export type AgentSessionStartOptions = { /** Hold `onEnter` until a participant joins the call. Defaults to `false`. */ waitForParticipant?: boolean; /** Keep the session running until shutdown is signaled. Defaults to `false`. */ runUntilShutdown?: boolean; /** Hang up the SIP leg when shutting down. Defaults to `false`. */ sipHangupOnShutdown?: boolean; /** Timeout in milliseconds for {@link AgentSessionStartOptions.waitForParticipant}. Defaults to `60000`. */ waitForParticipantTimeoutMs?: number; /** Also wait for the audio stream to be active before entering. Defaults to `false`. */ waitForAudioStream?: boolean; /** Extra runtime options forwarded to the cloud pipeline. Defaults to `null`. */ runtimeOptions?: Record | null; /** Caller-supplied session id to correlate the call. */ sessionId?: string; /** Override the Zero Runtime address to connect to. Defaults to the configured/env value. */ runtimeAddress?: string; /** Internal room configuration. @internal */ _roomConfig?: Record; /** Internal job context. @internal */ _jobCtx?: any; }; declare class AgentSessionImpl extends EventEmitter { agent: Agent; pipeline: Pipeline; onWakeUp: any; backgroundAudioConfig: any; backgroundAudio: any; currentUtterance: UtteranceHandle | null; /** Unresolved say()/reply() handles, oldest first (completion events carry no id). @internal */ private _pendingUtterances; /** In-flight deferred close scheduled from inside a tool call, or `null`. @internal */ private _deferredCloseTask; private _wakeUp; private _wakeUpMaxAttempts; private _wakeUpCount; private _wakeUpTask; private _wakeUpTaskCancelled; private readonly _wakeUpResetEvent; private _recording; private _recordingState; private _recordingManagerCache; private _userState; private _agentState; private _sessionId; private _signalingSessionId; private _grpcBridge; private _registry; private _registeredSessionId; private _a2aSubscribed; private _a2aHandler; private _a2aTopic; private _eventTask; private readonly _shutdownEvent; private _shutdownCallbacks; private _isSynthesizing; private readonly _synthesisDoneEvent; private _ttsCapabilities; private _lastEou; private _chatHistoryCache; private _transcriptMirror; private _sttObservationQueue; private _sttObservationTask; private readonly _sttObservedTexts; private _sttMutationWarned; private _thinkingAudioActive; private readonly _agentsById; private readonly _alternateIds; private readonly _handoffs; private _audioTrackCache; private _voicemailDetector; private _dtmfHandler; private _audioObserved; private readonly _audioObservedEvent; private _participantPresent; private readonly _participantArrivedEvent; private _audioStreamActive; private readonly _audioStreamActiveEvent; private _meetingJoinedEmitted; private _roomCallbacksBound; private _waitForParticipant; private _waitForParticipantTimeoutMs; private _sipHangupOnShutdown; private _waitForAudioStream; private _waitForAudioStreamExplicit; private _runtimeOptions; private _callerSessionId; private _jobCtx; private _boundJobCtx; constructor(agent: Agent, pipeline: Pipeline, opts?: AgentSessionOptions); /** Current {@link UserState} of the human participant. */ get userState(): UserState; /** Current {@link AgentState} of the agent. */ get agentState(): AgentState; /** Wake-up inactivity timeout in seconds, or `null` if disabled. Settable at runtime. */ get wakeUp(): number | null; set wakeUp(value: number | null); /** Signaling session id, when assigned. */ get signalingSessionId(): string | null; set signalingSessionId(value: string | null); /** Latest end-of-utterance result (e.g. `{ probability }`), or `null`. */ get lastEou(): Record | null; set lastEou(value: Record | null); /** Whether background audio is enabled for this call (per room options). */ get isBackgroundAudioEnabled(): boolean; /** Lazily-created audio track for the agent's outbound audio. */ get audioTrack(): unknown; /** Whether the agent is currently synthesizing/playing speech. */ get isSynthesizing(): boolean; /** Capabilities reported by the active TTS provider, or `null`. */ get ttsCapabilities(): Record | null; set ttsCapabilities(value: Record | null); /** Whether voicemail has been detected on the call. */ get isVoicemailDetected(): boolean; /** Latest recording status payload, or `null`. */ get recordingState(): Record | null; /** Helper exposing recording status (id, output URI, duration); see {@link _RecordingManager}. */ get recordingManager(): _RecordingManager; /** The agent's most recent spoken utterance metadata, or `null`. */ lastAgentSpeech: Record | null; /** Caller-supplied session id correlating the call. */ callerSessionId: string; /** Extra runtime options forwarded to the cloud pipeline. */ get runtimeOptions(): Record; /** Whether the session is configured to wait for a participant before entering. */ get waitForParticipant(): boolean; /** Participant-wait timeout in milliseconds. */ get waitForParticipantTimeoutMs(): number; /** Whether the SIP leg is hung up on shutdown. */ get sipHangupOnShutdown(): boolean; /** Whether the session waits for an active audio stream before entering. */ get waitForAudioStream(): boolean; /** Queue of observed STT events for an `stt` observation hook, or `null`. @internal */ get sttObservationQueue(): any; /** * Resolve once the agent has finished speaking. * @param timeout Optional timeout in seconds. * @returns `true` if synthesis completed, `false` if the timeout elapsed first. */ waitForSynthesisDone(timeout?: number): Promise; private _markSynthesisStarted; private _markSynthesisDone; private _mirrorUserTranscript; private _mirrorAgentGeneration; private static _extractMirrorText; private _autoStartThinkingAudio; private _autoStopThinkingAudio; /** The agent currently handling the call (changes across handoffs). */ activeAgent(): Agent; /** Look up a pre-registered agent by its id, or `null` if none matches. @internal */ agentById(agentId: string): Agent | null; private _seedHandoffRegistry; /** Register an agent as a valid handoff target and return its id. */ registerHandoffAgent(agent: Agent): string; /** * If a tool returned an {@link Agent}, convert it into a handoff directive; otherwise * return the result unchanged. */ interceptToolResult(result: any): any; /** Copy of the handoffs that have occurred this session, in order. */ handoffs(): AgentHandoff[]; /** Reason for the most recent handoff, or `''` if none. */ lastHandoffReason(): string; /** Swap the active agent to a registered target. @internal */ _applyAgentSwitch(from: string, to: string, reason: string): Promise; private _onDtmfReceived; private _onDtmfPubsub; private _onVoicemailDetected; /** Update the cached recording status and emit `recording_status_changed`. */ updateRecordingState(status: Record): void; /** * Connect the session to the Zero Runtime and run the agent. Optionally waits for a * participant and/or runs until shutdown. See {@link AgentSessionStartOptions}. */ start(opts?: AgentSessionStartOptions): Promise; private _startBound; private _checkSupportedHooks; private _maybeStartSttObservationPump; private static _sttEventText; /** Record a transcript text seen by an observe-only `stt` hook. @internal */ recordSttObservedText(event: any): void; private _detectSttMutation; /** * Speak `message` verbatim via TTS. * @param opts.interruptible Whether the user can interrupt playback. Defaults to `true`. * @param opts.voice Voice id/name override. Defaults to `''` (the configured voice). * @param opts.audioData Pre-synthesized audio (not supported; falls back to TTS). * @param opts.addToChatContext Append to transcript. Defaults to `true`. * @returns A handle to await or interrupt the utterance. */ say(message: string, opts?: { interruptible?: boolean; voice?: string; audioData?: any; addToChatContext?: boolean; [k: string]: any; }): Promise; /** * Generate and speak a model reply to `instructions`, optionally grounded on image frames. * @param opts.waitForPlayback Await playback completion before resolving. Defaults to `true`. * @param opts.frames Image frames to attach to the prompt. * @param opts.latestFrames Attach this many of the latest captured camera frames instead. * @param opts.interruptible Whether the user can interrupt. Defaults to `true`. * @returns A handle to await or interrupt the utterance. */ reply(instructions: string, opts?: { waitForPlayback?: boolean; frames?: any[]; latestFrames?: number; interruptible?: boolean; }): Promise; /** * Interrupt the agent's pending utterances and cancel any in-flight generation. * @param opts.force Interrupt even a non-interruptible utterance. Defaults to `false`. */ interrupt(opts?: { force?: boolean; }): void; /** Track a new utterance handle (FIFO; completion events carry no utterance id). @internal */ _registerUtterance(handle: UtteranceHandle): void; /** Resolve the oldest pending utterance handle (FIFO). @internal */ _resolveUtterance(opts?: { interrupted?: boolean; }): void; /** Resolve every pending utterance handle so no awaiting caller hangs. @internal */ _resolveAllUtterances(): void; private _scheduleRuntimeCancelGeneration; /** Register a callback to run when the session shuts down. */ addShutdownCallback(callback: () => unknown): void; /** Replace the active agent's instructions and apply the update to the live session. */ updateInstructions(instructions: string): Promise; /** Prompt the model to generate a response to `text` (spoken via the pipeline). */ generate(text: string): Promise; /** Cancel any in-flight model generation. */ cancelGeneration(): Promise; /** Enable or disable accepting user input (e.g. to mute the user during a prompt). */ setUserInputEnabled(enabled: boolean): Promise; /** Replace the active agent's tools and apply the update to the live session. */ updateTools(tools: any[]): Promise; /** * Return the conversation history. The result is an array of message records that is * also awaitable: use it synchronously for the cached snapshot, or `await` it to * fetch the latest from the live session. * @param opts.lastN Keep only the last N messages (`0` = all). Defaults to `0`. * @param opts.includeFunctionCalls Include tool/function messages. Defaults to `false`. * @param opts.includeSystemMessages Include system messages. Defaults to `false`. */ getContextHistory(opts?: { lastN?: number; includeFunctionCalls?: boolean; includeSystemMessages?: boolean; }): _HistoryAwaitable; /** * Fetch the conversation history from the live session (falling back to the local cache). * @param opts.lastN Keep only the last N messages (`0` = all). Defaults to `0`. * @param opts.includeFunctionCalls Include tool/function messages. Defaults to `false`. * @param opts.includeSystemMessages Include system messages. Defaults to `false`. */ fetchContextHistory(opts?: { lastN?: number; includeFunctionCalls?: boolean; includeSystemMessages?: boolean; }): Promise[]>; /** * Remove a message from the conversation history by id. * @returns `true` if the message was removed, `false` otherwise. */ removeMessage(messageId: string): Promise; /** * Insert a message into the conversation history out-of-band. * @param opts.position Optional placement hint for where to insert the message. * @returns `true` if the live session accepted the message. */ injectMessage(message: ChatMessage | { role: ChatRole | string; content: string; messageId?: string; images?: ImageContent[]; }, opts?: { position?: string; }): Promise; /** Inject every message from a {@link ChatContext}; returns `true` only if all succeeded. */ injectContext(ctx: ChatContext): Promise; /** * Fetch the conversation as a {@link ChatContext} from the live session. * @param opts.lastN Number of most-recent messages to fetch. Defaults to `0` (all). */ fetchChatContext(opts?: { lastN?: number; }): Promise; /** * Clear the running conversation history. With `keepSystemPrompt` (default `true`), * the agent's instructions are kept and only the turns are cleared. * @returns Whether the clear was applied. */ clearContext(opts?: { keepSystemPrompt?: boolean; }): Promise; /** * Retune the conversation context-window budget live (mid-session). Knobs mirror * {@link ContextWindow} and take effect on the next model turn; omitted knobs are * left unchanged. */ updateContextWindow(opts?: { maxTokens?: number; maxContextItems?: number; keepRecentTurns?: number; }): Promise; /** Summarize and compress the conversation now (fire-and-forget). */ summarizeContext(): Promise; /** Fetch the live conversation context and render it as OpenAI chat messages. */ contextToOpenai(opts?: { reasoningModel?: boolean; }): Promise>>; /** Fetch the live conversation context and render it as Anthropic Messages. */ contextToAnthropic(): Promise<{ messages: Array>; system: string | null; }>; /** Fetch the live conversation context and render it as Google Gemini contents. */ contextToGoogle(): Promise<{ contents: Array>; systemInstruction: string | null; }>; /** Fetch the live conversation context and serialize it to a plain object. */ contextToDict(): Promise>; private _resolveStub; private static _filterHistory; /** * End the session: run `onExit`, fire shutdown callbacks, disconnect, and clean up. * @param opts.reason Reason recorded for the close. Defaults to `'sdk_close'`. */ close(opts?: { reason?: string; }): Promise; /** Leave the call (closes the session with reason `'sdk_leave'`). */ leave(): Promise; /** Hang up the call (closes the session). */ hangup(reason?: string): Promise; /** * Resolve once a participant has joined the call, or the session shuts down first * (whichever happens sooner). Returns immediately if a participant is already present. */ waitForParticipantArrived(): Promise; /** * Swap a pipeline component's provider at runtime. * @param component Component to update (e.g. `'stt'`, `'llm'`, `'tts'`). * @param provider New provider identifier. * @param params Optional provider parameters. Defaults to `null`. */ updateProvider(component: string, provider: string, params?: Record | null): Promise; /** * Switch the whole pipeline to a different mode mid-session. * @param pipelineMode New pipeline mode (e.g. 'cascade', 'hybrid_stt'). * @param components Array of [component, provider, params] triples to reconfigure. */ changePipeline(pipelineMode: string, components: Array<[string, string, Record | null]>): Promise; /** * Tune interruption handling at runtime. Only the fields you set are applied. * @param opts.mode Detection mode: `'vad_only'`, `'stt_only'`, or `'hybrid'`. Throws if invalid. * @param opts.interruptMinDuration Minimum speech seconds to count as an interruption. * @param opts.interruptMinWords Minimum words to count as an interruption. * @param opts.cooldownMs Cooldown in milliseconds between interruptions. * @param opts.falseInterruptPauseDurationMs Pause in milliseconds after a false interruption. * @param opts.resumeOnFalseInterrupt Whether to resume speaking after a false interruption. */ updateInterruptConfig(opts?: { mode?: string; interruptMinDuration?: number; interruptMinWords?: number; cooldownMs?: number; falseInterruptPauseDurationMs?: number; resumeOnFalseInterrupt?: boolean; }): Promise; /** * Transfer the call to another destination (e.g. a SIP address). * @param token Auth token for the transfer. * @param transferTo Destination to transfer to; ignored if empty. */ callTransfer(token: string, transferTo: string): Promise; /** Not implemented; use `agentSwitch(...)` for multi-agent handoff. Always throws. */ warmTransfer(..._args: any[]): Promise; /** * Start recording the call. * @param config Recording config; falls back to the session-level `recording=` if omitted. */ startRecording(config?: any): Promise; /** Stop the active call recording. */ stopRecording(): Promise; /** * Play a background audio clip. `config` may be a URL/path string or a config object * with `file_url`/`volume`/`looping`. Set `enabled: false` to skip. */ playBackgroundAudio(config?: any, _overrideThinking?: boolean): Promise; /** Preload a background audio clip for low-latency playback. Accepts the same `config` as {@link playBackgroundAudio}. */ preloadBackgroundAudio(config?: any): Promise; /** Stop any background audio playback. */ stopBackgroundAudio(): Promise; private static _extractBgAudioArgs; /** * Push a raw PCM audio frame into the call (e.g. for custom input). * @param sampleRate Sample rate of the PCM data in Hz. Defaults to `48000`. */ pushAudioFrame(pcm: Buffer | Uint8Array, sampleRate?: number): Promise; /** * Send an image to the model for vision. Non-byte inputs are encoded to JPEG. * @param mimeType MIME type for raw bytes. Defaults to `'image/jpeg'`. */ sendImage(data: any, mimeType?: string): Promise; /** * Send a message together with image frames for a vision-grounded response. * @param frames Frames as `[bytes, mime]` tuples, captured vision frames, or raw bytes. * @param opts.defaultMimeType MIME used when a frame's type can't be inferred. Defaults to `'image/jpeg'`. * @param opts.numLatestFrames Attach this many of the latest captured frames as well. Defaults to `0`. */ sendMessageWithFrames(message: string | null, frames: any[], opts?: { defaultMimeType?: string; numLatestFrames?: number; }): Promise; /** * Publish a message on a pub/sub `topic` to other participants/agents. * @param config.topic Required topic name. * @param config.message Message body (stringified if not a string). * @param config.options Optional per-message options. * @param config.payload Optional structured payload. */ publishMessage(config: { topic: string; message?: unknown; options?: Record | null; payload?: unknown; }): Promise; /** * Subscribe to a pub/sub `topic`. The optional `callback` fires for matching messages. * @returns The registered handler (or `null`) so it can later be removed via * `session.off('pubsub_message', handler)`. */ subscribePubSub(topic: string, callback?: (msg: { topic: string; message: string; sender_id: string; sender_name: string; timestamp: string; payload: unknown; }) => void): Promise<((msg: any) => void) | null>; /** * Send an agent-to-agent (A2A) message to `targetAgentId`. * * A2A is delivered over pub/sub (topic `a2a/`): the message is published * as a JSON envelope `{ source_agent_id, message_json, correlation_id }`. * The receiving agent must have called {@link subscribeA2A} to receive it (surfaced as an * `a2a_message` event). */ sendA2A(targetAgentId: string, messageJson: string, correlationId?: string): Promise; /** * Subscribe the active agent to its agent-to-agent (A2A) topic so it receives * messages addressed to its id, surfaced as `a2a_message` events. */ subscribeA2A(): Promise; /** Start the agent's configured "thinking" audio loop (see `agent.setThinkingAudio`). */ startThinkingAudio(): Promise; /** Stop the "thinking" audio loop. */ stopThinkingAudio(): Promise; private _resetWakeUpTimer; private _startWakeUpWatcher; private _wakeUpWatcher; /** Set the {@link UserState} and emit `user_state_changed` (and `speech_in` when speaking). */ updateUserState(state: UserState): void; /** Set the {@link AgentState} and emit `agent_state_changed`. */ updateAgentState(state: AgentState): void; private _emitMeetingJoinedOnce; private _markParticipantArrived; private _markAudioStreamActive; private _firePreloadAssets; /** * Hold until a participant joins (when `waitForParticipant` is on). * @returns Whether `onEnter` should run: `false` when the session shut down * (e.g. the no-participant timeout) before anyone joined. @internal */ private _awaitParticipantIfNeeded; } /** * Read-only view of the current call recording's status, exposed via * {@link AgentSession.recordingManager}. */ export declare class _RecordingManager { /** Recording states considered active. */ static readonly ACTIVE_STATES: Set; private readonly _session; constructor(session: AgentSession); /** The latest recording status payload, or `null`. */ get lastStatus(): Record | null; /** Whether a recording is currently active. */ get isRecording(): boolean; /** Id of the current recording, or `''`. */ get recordingId(): string; /** Output URI of the recording, or `''`. */ get outputUri(): string; /** Track kind being recorded, or `''`. */ get trackKind(): string; /** Recorded duration in seconds. */ get durationSeconds(): number; /** Recorded file size in bytes. */ get fileSizeBytes(): number; } interface _HistoryFilters { lastN: number; includeFunctionCalls: boolean; includeSystemMessages: boolean; } /** Return type of {@link AgentSession.getContextHistory}: a message-record array that is also awaitable (await to fetch the latest). @internal */ export declare class _HistoryAwaitable extends Array> { private readonly _session; private readonly _filters; constructor(session: AgentSession, snapshot: Record[], filters: _HistoryFilters); static get [Symbol.species](): ArrayConstructor; then(onfulfilled?: ((value: Record[]) => TResult1 | PromiseLike) | null, onrejected?: ((reason: unknown) => TResult2 | PromiseLike) | null): Promise; } /** * A live voice call: binds an {@link Agent} to a {@link Pipeline}, connects to the Zero * Runtime, and drives the conversation. Provides call controls (`say`, `reply`, * `interrupt`, recording, background audio, transfers), conversation-history access, * pub/sub and agent-to-agent messaging, and emits {@link AgentSessionEvent}s. * * You normally don't create or start this yourself. Register an agent with * {@link serve}, then start a session against it with {@link invoke}; the serving * worker creates one `AgentSession` per dispatched call. Inside your {@link Agent}, * reach the running session through `this.session`. */ export type AgentSession = AgentSessionImpl; /** * Create an {@link AgentSession} for `agent` running on `pipeline`. * * Low-level constructor used internally by {@link serve} for each dispatched session. * Prefer {@link serve} + {@link invoke}; call this directly only when building a * custom worker that manages sessions itself. * @param opts See {@link AgentSessionOptions}. */ export declare const AgentSession: (agent: Agent, pipeline: Pipeline, opts?: AgentSessionOptions) => AgentSession; export {};