import { PersonaProtocolBase } from './base'; import { PersonaPacket, PersonaProtocolBaseConfig, ProtocolStatus, Session } from '../types'; type AudioAnalysisData = { localAmplitude: number; remoteAmplitude: number; }; type AudioVisualizerCallback = (data: AudioAnalysisData) => void; type PersonaWebRTCMessageCallback = (data: MessageEvent) => void; type PersonaWebRTCErrorCallback = (error: string) => void; type PersonaWebRTCConfig = PersonaProtocolBaseConfig & { webrtcUrl: string; iceServers?: RTCIceServer[]; }; // Core manda i server ICE, con le credenziali TURN, nel messaggio OPEN appena // il websocket รจ aperto. Un core che non lo manda lascia solo lo STUN. const DEFAULT_ICE_SERVERS: RTCIceServer[] = [{ urls: 'stun:stun.cloudflare.com:3478' }]; const ICE_SERVERS_TIMEOUT_MS = 3000; class PersonaWebRTCClient { private config: PersonaWebRTCConfig; private pc: RTCPeerConnection | null = null; private ws: WebSocket | null = null; private localStream: MediaStream | null = null; private remoteStream: MediaStream = new MediaStream(); private audioCtx: AudioContext | null = null; private localAnalyser: AnalyserNode | null = null; private remoteAnalyser: AnalyserNode | null = null; private analyzerFrame: number | null = null; private dataChannel: RTCDataChannel | null = null; private isConnected: boolean = false; private visualizerCallbacks: AudioVisualizerCallback[] = []; private messageCallbacks: PersonaWebRTCMessageCallback[] = []; private errorCallbacks: PersonaWebRTCErrorCallback[] = []; private queuedMessages: PersonaPacket[] = []; private onIceServers: ((iceServers: RTCIceServer[] | null) => void) | null = null; constructor(config: PersonaWebRTCConfig) { this.config = config; } public async connect(session: Session): Promise { if (this.isConnected) return; this.isConnected = true; try { this.localStream = await navigator.mediaDevices.getUserMedia({ audio: true }); } catch (err) { this.config.logger?.error('Error accessing microphone:', err); return; } const url = this.config.webrtcUrl || 'wss://persona.applica.guru/api/webrtc'; this.ws = new WebSocket(`${url}?apiKey=${encodeURIComponent(this.config.apiKey)}`); const serverIceServers = this._waitForIceServers(this.ws); this.ws.onmessage = async (event) => { const data = JSON.parse(event.data); if (data.type === 'OPEN') { this.onIceServers?.(data.payload?.iceServers || null); } else if (data.type === 'ANSWER') { await this.pc!.setRemoteDescription(new RTCSessionDescription(data.payload.sdp)); } else if (data.type === 'CANDIDATE') { try { await this.pc!.addIceCandidate(new RTCIceCandidate(data.payload.candidate)); } catch (err) { this.config.logger?.error('Error adding ICE candidate:', err); } } }; this.ws.onclose = (event: CloseEvent) => { if (event.code !== 1000) { this.errorCallbacks.forEach((callback) => { callback('Oops! The connection to the server was lost. Please try again later.'); }); } this._stopAnalyzers(); }; const iceServers = this.config.iceServers || (await serverIceServers) || DEFAULT_ICE_SERVERS; if (!this.isConnected || this.ws.readyState !== WebSocket.OPEN) return; this.pc = new RTCPeerConnection({ iceServers }); this.localStream.getTracks().forEach((track) => { this.pc!.addTrack(track, this.localStream!); }); this.pc.ontrack = (event) => { event.streams[0].getTracks().forEach((track) => { this.remoteStream.addTrack(track); }); if (!this.audioCtx) { this._startAnalyzers(); } const remoteAudio = new Audio(); remoteAudio.srcObject = this.remoteStream; remoteAudio.play().catch((e) => { this.config.logger?.error('Error playing remote audio:', e); }); }; this.pc.onicecandidate = (event) => { if (event.candidate && this.ws?.readyState === WebSocket.OPEN) { this.ws.send( JSON.stringify({ type: 'CANDIDATE', src: 'client', payload: { candidate: event.candidate }, }), ); } }; this.pc.ondatachannel = (event) => { const channel = event.channel; channel.onmessage = (msg) => { this.messageCallbacks.forEach((callback) => { callback(msg); }); }; channel.onopen = () => { while (this.queuedMessages.length > 0) { const packet = this.queuedMessages.shift(); if (packet) { channel.send(JSON.stringify(packet)); this.config.logger?.info('Sent queued message:', packet); } } }; }; // Il datachannel deve esistere prima dell'offerta, o non entra nell'SDP. this.createDataChannel(); const offer = await this.pc.createOffer(); await this.pc.setLocalDescription(offer); const metadata = { apiKey: this.config.apiKey, agentId: this.config.agentId, userId: this.config.userId || 'anonymous', sessionCode: session as string, }; this.config.logger?.debug('Opening connection to WebRTC server: ', metadata); const offerMessage = { type: 'OFFER', src: crypto.randomUUID?.() || 'client_' + Date.now(), payload: { sdp: { sdp: offer.sdp, type: offer.type, }, connectionId: (Date.now() % 1000000).toString(), metadata, }, }; this.ws.send(JSON.stringify(offerMessage)); } /** * Server ICE annunciati da core nel messaggio OPEN; null se il socket si * chiude prima o se non arrivano entro il timeout dall'apertura (core senza * OPEN). */ private _waitForIceServers(ws: WebSocket): Promise { return new Promise((resolve) => { let timer: ReturnType | undefined; const done = (iceServers: RTCIceServer[] | null) => { clearTimeout(timer); ws.removeEventListener('open', onOpen); ws.removeEventListener('close', onClose); this.onIceServers = null; resolve(iceServers); }; const onOpen = () => { timer = setTimeout(() => done(null), ICE_SERVERS_TIMEOUT_MS); }; const onClose = () => done(null); ws.addEventListener('open', onOpen); ws.addEventListener('close', onClose); this.onIceServers = done; }); } public async disconnect(): Promise { if (!this.isConnected) return; this.isConnected = false; if (this.ws?.readyState === WebSocket.OPEN) this.ws.close(); if (this.pc) this.pc.close(); this.pc = null; this.dataChannel = null; if (this.localStream) { this.localStream.getTracks().forEach((track) => track.stop()); } this.remoteStream = new MediaStream(); if (this.audioCtx) { await this.audioCtx.close(); this.audioCtx = null; } this._stopAnalyzers(); } public addVisualizerCallback(callback: AudioVisualizerCallback): void { this.visualizerCallbacks.push(callback); } public addMessageCallback(callback: PersonaWebRTCMessageCallback): void { this.messageCallbacks.push(callback); } public addErrorCallback(callback: PersonaWebRTCErrorCallback): void { this.errorCallbacks.push(callback); } public createDataChannel(label = 'messages'): void { if (!this.pc || this.dataChannel) return; this.dataChannel = this.pc.createDataChannel(label); this.dataChannel.onopen = () => { this.config.logger?.info('Data channel opened'); while (this.queuedMessages.length > 0) { const packet = this.queuedMessages.shift(); if (packet) { this.dataChannel!.send(JSON.stringify(packet)); this.config.logger?.info('Sent queued message:', packet); } } }; this.dataChannel.onmessage = (msg: MessageEvent) => { this.messageCallbacks.forEach((callback) => { callback(msg); }); }; } public sendPacket(packet: PersonaPacket): void { if (!this.dataChannel || this.dataChannel.readyState !== 'open') { this.queuedMessages.push(packet); return; } this.dataChannel.send(JSON.stringify(packet)); this.config.logger?.info('Sent message:', packet); } private _startAnalyzers(): void { if (!this.localStream || !this.remoteStream || this.visualizerCallbacks.length === 0) { return; } // eslint-disable-next-line @typescript-eslint/no-explicit-any this.audioCtx = new (window.AudioContext || (window as any).webkitAudioContext)(); const localSource = this.audioCtx.createMediaStreamSource(this.localStream); const remoteSource = this.audioCtx.createMediaStreamSource(this.remoteStream); this.localAnalyser = this.audioCtx.createAnalyser(); this.remoteAnalyser = this.audioCtx.createAnalyser(); this.localAnalyser.fftSize = 256; this.remoteAnalyser.fftSize = 256; localSource.connect(this.localAnalyser); remoteSource.connect(this.remoteAnalyser); const loop = () => { if (!this.localAnalyser || !this.remoteAnalyser || this.visualizerCallbacks.length === 0) { return; } const localArray = new Uint8Array(this.localAnalyser.frequencyBinCount); const remoteArray = new Uint8Array(this.remoteAnalyser.frequencyBinCount); this.localAnalyser.getByteFrequencyData(localArray); this.remoteAnalyser.getByteFrequencyData(remoteArray); const localAmp = localArray.reduce((a, b) => a + b, 0) / localArray.length; const remoteAmp = remoteArray.reduce((a, b) => a + b, 0) / remoteArray.length; if (this.visualizerCallbacks.length > 0) { this.visualizerCallbacks.forEach((callback) => { callback({ localAmplitude: localAmp, remoteAmplitude: remoteAmp, }); }); } this.analyzerFrame = requestAnimationFrame(loop); }; this.analyzerFrame = requestAnimationFrame(loop); } private _stopAnalyzers(): void { if (this.analyzerFrame) { cancelAnimationFrame(this.analyzerFrame); this.analyzerFrame = null; } this.localAnalyser = null; this.remoteAnalyser = null; } } type PersonaWebRTCProtocolConfig = PersonaWebRTCConfig & { autostart?: boolean; }; class PersonaWebRTCProtocol extends PersonaProtocolBase { status: ProtocolStatus; session: Session; autostart: boolean; config: PersonaWebRTCProtocolConfig; webRTCClient: PersonaWebRTCClient; constructor(config: PersonaWebRTCProtocolConfig) { super(); this.config = config; this.status = 'disconnected'; this.session = null; this.autostart = config?.autostart ?? false; this.webRTCClient = new PersonaWebRTCClient(config); this.webRTCClient.addMessageCallback((msg: MessageEvent) => { const data = JSON.parse(msg.data) as PersonaPacket; this.notifyPacket(data); }); this.webRTCClient.addErrorCallback((error: string) => { this.config.logger?.error('WebRTC error:', error); this.notifyPacket({ type: 'message', payload: { type: 'text', role: 'assistant', text: error, }, }); }); } public getName(): string { return 'webrtc'; } public getPriority(): number { return 1000; } public async syncSession(session: Session): Promise { super.syncSession(session); if (this.status === 'connected') { await this.disconnect(); await this.connect(session); } } public async connect(session?: Session): Promise { if (this.status === 'connected') { return Promise.resolve(this.session); } this.session = session || this.session || 'new'; this.setStatus('connecting'); this.config.logger?.debug('Connecting to WebRTC with sessionId:', this.session); await this.webRTCClient.connect(this.session); this.setStatus('connected'); await this.webRTCClient.createDataChannel(); return this.session; } public async disconnect(): Promise { if (this.status === 'disconnected') { this.config.logger?.warn('Already disconnected'); return Promise.resolve(); } await this.webRTCClient.disconnect(); this.setStatus('disconnected'); this.config?.logger?.debug('Disconnected from WebRTC'); } public sendPacket(packet: PersonaPacket): Promise { if (this.status !== 'connected') { return Promise.reject(new Error('Not connected')); } this.webRTCClient.sendPacket(packet); return Promise.resolve(); } } export { PersonaWebRTCProtocol }; export type { PersonaWebRTCProtocolConfig, AudioVisualizerCallback, AudioAnalysisData };