import { Disposable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/lifecycle"; import { IObservable } from "@codingame/monaco-vscode-api/vscode/vs/base/common/observable"; import { URI } from "@codingame/monaco-vscode-api/vscode/vs/base/common/uri"; import { ICommandService } from "@codingame/monaco-vscode-api/vscode/vs/platform/commands/common/commands.service"; import { ILogService } from "@codingame/monaco-vscode-api/vscode/vs/platform/log/common/log.service"; import { IAuthenticationService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/services/authentication/common/authentication.service"; import { IVoiceTranscriptStore } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/agentsVoice/common/voiceTranscriptStore.service"; import { IVoiceClientService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/common/voiceClient/voiceClientService.service"; import { IMicCaptureService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/micCaptureService.service"; import { ITtsPlaybackService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/ttsPlaybackService.service"; import { IVoiceToolDispatchService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/voiceToolDispatchService.service"; import { IVoicePlaybackService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/common/voicePlaybackService.service"; import { IAgentSessionsService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/agentSessions/agentSessionsService.service"; import { IChatService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/common/chatService/chatService.service"; import { IWorkbenchEnvironmentService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/services/environment/common/environmentService.service"; import { ITelemetryService } from "@codingame/monaco-vscode-api/vscode/vs/platform/telemetry/common/telemetry.service"; import { IConfigurationService } from "@codingame/monaco-vscode-api/vscode/vs/platform/configuration/common/configuration.service"; import { IAccessibilitySignalService } from "@codingame/monaco-vscode-api/vscode/vs/platform/accessibilitySignal/browser/accessibilitySignalService.service"; import { IAccessibilityService } from "@codingame/monaco-vscode-api/vscode/vs/platform/accessibility/common/accessibility.service"; import { IVoiceSessionController } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/browser/voiceClient/voiceSessionController.service"; export type VoiceState = "idle" | "listening" | "processing" | "speaking" | "error"; export interface IPendingToolConfirmation { readonly type: "approval" | "input"; readonly sessionLabel: string; readonly sessionResource: URI; readonly description: string; approve(): void; deny(): void; } export interface ITranscriptTurn { readonly speaker: "user" | "assistant"; readonly text: string; /** Stable-recognition prefix of `text`. User turns only; empty otherwise. */ readonly committed: string; /** True while the user is still speaking (live recognition). */ readonly isPartial: boolean; } export declare class VoiceSessionController extends Disposable implements IVoiceSessionController { private readonly voiceClientService; private readonly micCaptureService; private readonly ttsPlaybackService; private readonly voiceToolDispatchService; private readonly voicePlaybackService; private readonly agentSessionsService; private readonly chatService; private readonly commandService; private readonly authenticationService; private readonly voiceTranscriptStore; private readonly logService; private readonly environmentService; private readonly telemetryService; private readonly configurationService; private readonly accessibilitySignalService; private readonly accessibilityService; readonly _serviceBrand: undefined; private readonly _voiceState; readonly voiceState: IObservable; private readonly _statusText; readonly statusText: IObservable; private static readonly _MAX_TURNS; private readonly _transcriptTurns; readonly transcriptTurns: IObservable; private readonly _isConnected; readonly isConnected: IObservable; private readonly _isConnecting; readonly isConnecting: IObservable; private readonly _isReconnecting; readonly isReconnecting: IObservable; private readonly _pendingToolConfirmations; readonly pendingToolConfirmations: IObservable; private readonly _targetSession; readonly targetSession: IObservable; private _pttHeld; private _pttToggleMode; private _pttCurrentTurnId; private _window; private readonly _voiceEventDisposables; private readonly _voiceAutorunDisposable; private readonly _autoApprovedSessions; private _transcriptFadeTimer; private _pttMaxDurationTimer; private static readonly _PTT_MAX_DURATION_MS; /** Short-tap threshold: if the key is held for less than this, enter * toggle mode where a second tap finishes the recording. */ private static readonly _PTT_TOGGLE_THRESHOLD_MS; /** Debounce before re-entering listening after assistant stops speaking. */ private static readonly _AUTO_LISTEN_QUIET_MS; private _delayedMicStopTimer; private _autoSendSilenceTimer; private _autoListenTimer; private _pttWaitingForPlayback; /** Guards auto re-listen: only re-arm after a reply has actually played. */ private _replyPlayedSinceSend; /** Set after send_to_chat; blocks auto-listen until the reply TTS starts. */ private _awaitingReplyAudio; private _awaitingReplyWatchdog; /** Enter listening immediately after greeting finishes (no debounce). */ private _autoListenAfterGreeting; /** Tracks whether the initial listen cue has been played after connecting. */ private _hasPlayedInitialListenCue; private readonly _audioQueue; private _currentPlaybackSessionId; private _isProcessingQueue; private _suppressIncomingAudio; private readonly _sessionAudioCache; private _replaySourceNode; private readonly _prevSessionStates; private readonly _userCancelledSessions; private static readonly _USER_CANCEL_SUPPRESS_MS; private readonly _confirmationFlushWatchdogs; private static readonly _CONFIRMATION_FLUSH_DELAY_MS; /** Model refs eagerly loaded for sessions awaiting input (no UI focus needed). */ private readonly _eagerModelRefs; /** Sessions with an in-flight eager model load, to dedupe concurrent loads. */ private readonly _eagerModelLoading; /** * Sessions whose ``idle`` transition is being deferred until their chat * model loads, so the narration can include ``last_response_summary``. * While a session id is in this set we suppress emitting a premature, * summary-less ``idle`` to the backend (see _buildSessionContext). */ private readonly _pendingIdleNarration; private _telemetrySessionIndex; private _telemetrySessionStart; private _telemetryTurnCount; private _telemetryReconnectCount; private _telemetryFirstConnect; private _telemetryConnectStartMs; private _telemetryLastConnectMs; private _telemetryPttDownMs; private _telemetryPttUpMs; private _telemetryFirstTranscriptionMs; private _telemetryTtsInterrupted; /** Cached GitHub login resolved on connect; used as transcript partition key. */ private _userLogin; /** Locally-persisted turn id of the last assistant turn we appended. * Used as the ancestor of the next user turn we persist. */ private _lastPersistedTurnId; /** Last-N cross-session timeline entries — voice turns, voice tool * calls, coding-session events, plus a synthesized first-2-sentences * summary of the latest Copilot reply per active session. Sent to the * BE on the next start_session and then cleared — single-shot recall. */ private _pendingPriorTimeline; /** * How many of the most recent persisted timeline entries we forward * to the BE (across all kinds). Coding-agent reply synthesis happens * on top of this — we add one entry per active coding session. */ private static readonly PRIOR_TIMELINE_ENTRY_LIMIT; /** * Max sentences of Copilot's last reply we include per active coding * session when synthesizing ``coding_agent_reply`` entries. Bounded * because the full reply can be arbitrarily long. */ private static readonly CODING_AGENT_REPLY_SENTENCE_LIMIT; constructor(voiceClientService: IVoiceClientService, micCaptureService: IMicCaptureService, ttsPlaybackService: ITtsPlaybackService, voiceToolDispatchService: IVoiceToolDispatchService, voicePlaybackService: IVoicePlaybackService, agentSessionsService: IAgentSessionsService, chatService: IChatService, commandService: ICommandService, authenticationService: IAuthenticationService, voiceTranscriptStore: IVoiceTranscriptStore, logService: ILogService, environmentService: IWorkbenchEnvironmentService, telemetryService: ITelemetryService, configurationService: IConfigurationService, accessibilitySignalService: IAccessibilitySignalService, accessibilityService: IAccessibilityService); connect(window: Window & typeof globalThis): Promise; disconnect(): void; /** DEV ONLY: Simulate a connected session with fake transcript for UI testing. */ simulateConnection(): void; private _onConnectionLost; pttDown(): void; pttUp(): void; private _finishPtt; markUserCancelled(sessionId: string): void; setTargetSession(resource: URI | undefined): void; newSessionAsTarget(): void; private _scheduleDelayedMicStop; private _isAutoSendEnabled; /** * Check if the transcript text ends with the configured send keyword. * Returns the text with the keyword stripped if found, or undefined if not. */ private _checkSendKeyword; /** Re-enter listening via synthetic short tap. */ private _enterAutoListen; /** Debounced re-listen after assistant stops speaking. */ private _scheduleAutoListen; private _clearAutoListenTimer; /** Auto-finish recording after configured silence in toggle mode. */ private _scheduleAutoSendOnSilence; private _clearAutoSendSilenceTimer; /** Block auto-listen until reply audio arrives (with 30s watchdog). */ private _setAwaitingReply; private _clearAwaitingReply; /** * Send transcription text to the target session or active chat. * If a target session is selected, sends directly via chatService. * Otherwise sends to whatever is currently active via the view pane command. */ private _sendTranscriptionToChat; /** * Watch a session's latest response and surface it in the floating window * transcript. Called when voice sends to a non-visible session so the user * can see the reply without switching the chat panel. */ private _watchResponseForFloatingWindow; private _pushTurn; /** * Start a new user turn at the tail of the buffer. If the previous tail is * already an empty user turn (rapid PTT toggle before any transcription * landed), reuse it instead of pushing a duplicate empty entry. */ private _startUserTurn; private _updateUserTurn; /** * Update the assistant turn at the tail of the buffer with `text`. * * The streaming TTS pipeline pushes a monotonically-growing transcript * with each audio chunk of a response. `startNewTurn` distinguishes * the first chunk of a NEW response (push a fresh assistant turn) * from continuation chunks of the SAME response (replace the tail's * text as the transcript grows). This prevents two distinct * assistant responses from collapsing into one when they happen * back-to-back without an intervening user turn (e.g. proactive * narration followed by a command reply). */ private _setAssistantTurn; private _cancelTranscriptFade; /** * Append a final entry to the on-disk transcript store. * * Entry ids are generated locally — voice_code's backend has no persistent * conversation memory today, so there's no server-issued id to defer to. * Each new entry chains off the previous one via ``ancestorIds`` so a UI * can show the linear conversation order. * * ``user_voice`` and ``agent_voice`` are user-visible in the transcripts * pane. ``agent_tool_call`` and ``coding_event`` are persisted only so we * can replay them as cross-session context to the backend on reconnect. */ private _persistEntry; /** Back-compat thin shim for the two existing voice call sites. */ private _persistTurn; /** * One-line, human/LLM-readable summary of a voice tool call for the * timeline. Backend's prior_timeline renderer expects this format — keep * it stable. * * send_to_chat(text="Open a new terminal and cd into the current directory.") * new_sessions(sessions=[{"text": "Refactor upload service"}]) * approve_confirmation(...) */ private _renderToolCallSummary; /** * Convert persisted transcript turns into typed timeline entries for * the BE, then top up with a synthesized ``coding_agent_reply`` per * active coding session (first ~2 sentences of the latest Copilot * response). The synthetic entries are *not* persisted — they read * live ``IChatModel`` state so the summary stays fresh on every * reconnect. * * Output is chronological (oldest first), matching what the BE * renders into its ``[PRIOR_CONTEXT]`` block. Synthetic * ``coding_agent_reply`` entries are appended at the end since they * represent the *current* state of coding sessions at reconnect. */ private _buildPriorTimeline; /** * Return the first ``n`` sentences of ``text``. Cheap regex split — * good enough for a prompt-prefix summary; we don't need perfect NLP * boundaries here. Falls back to a hard char cap if no terminator * shows up in the first 600 chars. */ private _firstSentences; private _enqueueAudio; private _playChunk; private _processQueue; private _replaySessionAudio; private _stopReplay; private _sendContext; /** * Paranoid mitigation for the "confirmation narration not fired while user * is on the same session" symptom. Even though the autorun calls * `_sendContext + flushSessionContext` at the transition, in practice * users observed that the BE-side narration ("I need approval to run X") * only fires after they navigate AWAY from the session. * * As a guarded re-flush we schedule a single delayed `_sendContext + flush` * per session that's awaiting confirmation. The merge-patch in * `_sendDelta` short-circuits when no fields changed (see lines 393-395), * so a no-op re-send is silent on the BE — but if the FIRST send was * dropped (race condition, debounce hiccup, WS coalescing), this second * send pushes the state through. * * The watchdog auto-clears once the autorun observes the session has left * `waiting_for_confirmation`. */ private _armConfirmationFlushWatchdog; /** * Check all sessions for state changes and send notifications to backend. * This catches state transitions for sessions without a loaded chat model * (which the autorun can't track via observables), and also regular chat * sessions that are not agent sessions. */ private _checkSessionStateChanges; private _buildSessionContext; /** * Eagerly load a chat model for a session that needs input but hasn't been * opened in the UI yet. Once loaded, the autorun observables will re-fire * with full confirmation detail so the backend can narrate properly. */ private _ensureModelLoaded; /** * Defer narrating a session's ``idle`` transition until its chat model is * resident, so the narration can include ``last_response_summary``. Remote/ * Copilot sessions don't keep their model loaded, so without this the * backend would only ever see a summary-less completion. Eagerly loads the * model; once it resolves the autorun re-fires and narrates with the summary. */ private _deferIdleNarrationUntilModelLoaded; private _getAgentStateInfo; private _classifyPendingType; private _getConfirmationDescription; private _autoApproveCheck; private _getMachineId; submitFeedback(feedbackText: string): Promise<{ ok: boolean; error?: string; }>; }