// ============================================================================ // ElevenLabs Text-to-Speech helper // // Uses the /with-timestamps API endpoint to get character-level alignment data // alongside audio. This enables zero-API-call text-selection playback by // seeking within cached full-text audio. // ============================================================================ import { getSettings } from "./settings"; // --- Markdown stripping --------------------------------------------------- /** Strip Markdown syntax to produce clean plain text for TTS */ export function stripMarkdown(md: string): string { return ( md // Remove code blocks (triple backtick) .replace(/```[\s\S]*?```/g, "") // Remove inline code .replace(/`([^`]+)`/g, "$1") // Remove images ![alt](url) .replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1") // Convert links [text](url) to just text .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // Remove headers (# ## ### etc) .replace(/^#{1,6}\s+/gm, "") // Remove bold/italic markers .replace(/(\*{1,3}|_{1,3})(.*?)\1/g, "$2") // Remove strikethrough .replace(/~~(.*?)~~/g, "$1") // Remove horizontal rules .replace(/^[-*_]{3,}\s*$/gm, "") // Remove blockquotes .replace(/^>\s+/gm, "") // Remove list markers .replace(/^[\s]*[-*+]\s+/gm, "") .replace(/^[\s]*\d+\.\s+/gm, "") // Remove HTML tags .replace(/<[^>]+>/g, "") // Collapse multiple newlines .replace(/\n{3,}/g, "\n\n") .trim() ); } // --- Types ---------------------------------------------------------------- /** Character-level alignment from ElevenLabs with-timestamps API */ export interface CharacterAlignment { characters: string[]; characterStartTimesSeconds: number[]; characterEndTimesSeconds: number[]; } /** Cached audio entry with alignment data for seek-based selection playback */ export interface CachedAudio { blob: Blob; alignment: CharacterAlignment; strippedText: string; // exact text sent to ElevenLabs } export type TtsState = "idle" | "loading" | "playing" | "paused" | "error"; // --- Audio cache (session-scoped, LRU, max 10 entries) -------------------- const audioCache = new Map(); const CACHE_MAX = 10; function cacheKey(voiceId: string, model: string, text: string): string { return `${voiceId}:${model}:${text}`; } function cacheGet(key: string): CachedAudio | undefined { const entry = audioCache.get(key); if (entry) { // LRU: move to end by delete + re-insert audioCache.delete(key); audioCache.set(key, entry); } return entry; } function cachePut(key: string, entry: CachedAudio): void { if (audioCache.size >= CACHE_MAX) { // Evict oldest (first key in Map iteration order) const oldest = audioCache.keys().next().value; if (oldest) audioCache.delete(oldest); } audioCache.set(key, entry); } // --- Substring alignment lookup ------------------------------------------- /** Normalize whitespace for fuzzy substring matching */ function normalizeWhitespace(s: string): string { return s.replace(/\s+/g, " ").trim().toLowerCase(); } /** * Check if any cached audio contains the given text as a substring. * If found, returns the blob and start/end times for seeking. * This enables zero-API-call playback for text selections. */ export function findCachedAlignmentForText(selectedText: string): { blob: Blob; startTime: number; endTime: number; } | null { const normalizedSelection = normalizeWhitespace(selectedText); if (!normalizedSelection) return null; for (const entry of audioCache.values()) { const normalizedFull = normalizeWhitespace(entry.strippedText); const idx = normalizedFull.indexOf(normalizedSelection); if (idx === -1) continue; // Build a mapping from each char in strippedText to its position in the // normalized (lowercased, collapsed-whitespace) version. const { strippedText, alignment } = entry; const charToNormPos: number[] = []; let np = 0; let prevWasSpace = false; for (let i = 0; i < strippedText.length; i++) { const isSpace = /\s/.test(strippedText[i]); if (isSpace) { if (!prevWasSpace && np > 0) { charToNormPos.push(np); np++; } else { charToNormPos.push(-1); // collapsed away } prevWasSpace = true; } else { charToNormPos.push(np); np++; prevWasSpace = false; } } // Find the strippedText char indices that map to the normalized range const selEnd = idx + normalizedSelection.length - 1; let startCharIdx = -1; let endCharIdx = -1; for (let i = 0; i < charToNormPos.length; i++) { if (charToNormPos[i] === idx && startCharIdx === -1) { startCharIdx = i; } if (charToNormPos[i] === selEnd) { endCharIdx = i; } } if (startCharIdx === -1 || endCharIdx === -1) continue; // Clamp to alignment array bounds const alignLen = alignment.characterStartTimesSeconds.length; if (alignLen === 0) continue; const clampedStart = Math.min(startCharIdx, alignLen - 1); const clampedEnd = Math.min(endCharIdx, alignment.characterEndTimesSeconds.length - 1); if (clampedStart < 0 || clampedEnd < 0) continue; return { blob: entry.blob, startTime: alignment.characterStartTimesSeconds[clampedStart], endTime: alignment.characterEndTimesSeconds[clampedEnd], }; } return null; } // --- Audio playback state ------------------------------------------------- /** Current audio element for stop control */ let currentAudio: HTMLAudioElement | null = null; let currentBlobUrl: string | null = null; /** When playing a selection from cached audio, auto-stop at this time */ let selectionEndTime: number | null = null; // --- Playback controls ---------------------------------------------------- /** Set playback speed on the current audio element */ export function setTtsPlaybackRate(rate: number): void { if (currentAudio) { currentAudio.playbackRate = rate; } } /** Pause TTS audio without destroying — preserves position for resume */ export function pauseTts(): void { if (currentAudio && !currentAudio.paused) { currentAudio.pause(); } } /** Resume paused TTS audio from where it left off */ export function resumeTts(): void { if (currentAudio && currentAudio.paused && currentAudio.currentTime > 0) { currentAudio.play(); } } /** Stop any currently playing TTS audio */ export function stopTts(): void { if (currentAudio) { currentAudio.pause(); currentAudio.currentTime = 0; currentAudio.onended = null; currentAudio.onerror = null; currentAudio.ontimeupdate = null; currentAudio = null; } if (currentBlobUrl) { URL.revokeObjectURL(currentBlobUrl); currentBlobUrl = null; } selectionEndTime = null; } // --- Internal: create Audio from blob and play ---------------------------- /** Internal: create Audio from blob and play it */ function playBlobAsAudio( blob: Blob, onStateChange: (state: TtsState, error?: string) => void, ): void { // Full-text playback: no auto-stop selectionEndTime = null; const blobUrl = URL.createObjectURL(blob); currentBlobUrl = blobUrl; const audio = new Audio(blobUrl); currentAudio = audio; onStateChange("playing"); audio.onended = () => { stopTts(); onStateChange("idle"); }; audio.onerror = () => { stopTts(); onStateChange("error", "Audio playback failed"); }; audio.play(); } // --- Main TTS functions --------------------------------------------------- /** * Call ElevenLabs TTS API (with-timestamps) and play the result. * Uses cache to avoid redundant API calls. * * @param text - Plain text to speak (already stripped of Markdown) * @param onStateChange - Callback for state transitions */ export async function speakWithElevenLabs( text: string, onStateChange: (state: TtsState, error?: string) => void, ): Promise { // Stop any existing playback stopTts(); const settings = getSettings(); if (!settings.elevenLabsApiKey) { onStateChange("error", "No API key configured"); return; } const voiceId = settings.elevenLabsVoiceId; const key = cacheKey(voiceId, settings.elevenLabsModel, text); // Check cache first — zero API call on hit const cached = cacheGet(key); if (cached) { playBlobAsAudio(cached.blob, onStateChange); return; } onStateChange("loading"); try { const res = await fetch( `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/with-timestamps?output_format=mp3_44100_128`, { method: "POST", headers: { "Content-Type": "application/json", "xi-api-key": settings.elevenLabsApiKey, }, body: JSON.stringify({ text, model_id: settings.elevenLabsModel, }), }, ); if (!res.ok) { const errorText = await res.text().catch(() => res.statusText); throw new Error(`ElevenLabs API error (${res.status}): ${errorText}`); } const json = await res.json(); const { audio_base64, alignment } = json; // Decode base64 to Blob const binary = atob(audio_base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); const blob = new Blob([bytes], { type: "audio/mpeg" }); // Store in cache with alignment data const cachedEntry: CachedAudio = { blob, alignment: { characters: alignment.characters, characterStartTimesSeconds: alignment.character_start_times_seconds, characterEndTimesSeconds: alignment.character_end_times_seconds, }, strippedText: text, }; cachePut(key, cachedEntry); playBlobAsAudio(blob, onStateChange); } catch (err) { stopTts(); const message = err instanceof Error ? err.message : "TTS failed"; onStateChange("error", message); } } /** * Play a text selection, leveraging cached audio when possible. * * Credit-saving flow: * 1. Check if any cached full-text audio contains this selection as a substring * 2. If yes: create Audio from cached blob, seek to start time, auto-stop at end time * 3. If no: fall back to speakWithElevenLabs() (makes API call, result gets cached) * * @param selectedText - The raw text the user selected (from window.getSelection().toString()) * @param onStateChange - State change callback (same as speakWithElevenLabs) */ export async function speakSelection( selectedText: string, onStateChange: (state: TtsState, error?: string) => void, ): Promise { // Stop any existing playback first stopTts(); // Try zero-API-call path: seek within cached full-text audio const cached = findCachedAlignmentForText(selectedText); if (cached) { const { blob, startTime, endTime } = cached; const blobUrl = URL.createObjectURL(blob); currentBlobUrl = blobUrl; const audio = new Audio(blobUrl); currentAudio = audio; selectionEndTime = endTime; audio.onended = () => { stopTts(); onStateChange("idle"); }; audio.onerror = () => { stopTts(); onStateChange("error", "Audio playback failed"); }; // Auto-stop at selection end time audio.ontimeupdate = () => { if (selectionEndTime !== null && audio.currentTime >= selectionEndTime) { stopTts(); onStateChange("idle"); } }; // Seek to selection start, then play audio.currentTime = startTime; onStateChange("playing"); await audio.play(); return; } // Cache miss: fall back to full API call for just the selected text // This result also gets cached (with its own alignment data) for future replays await speakWithElevenLabs(selectedText, onStateChange); }