/** * Represents a single mouth shape with timing information */ interface MouthCue { /** Start time in seconds */ start: number; /** End time in seconds */ end: number; /** Mouth shape: X (closed), A-H (various open positions) */ value: string; } /** * Result of lip-sync-engine analysis */ interface LipSyncEngineResult { /** Array of mouth cues with precise timing */ mouthCues: MouthCue[]; /** Optional metadata about the analysis */ metadata?: { /** Duration of analyzed audio in seconds */ duration: number; /** Sample rate of input audio */ sampleRate: number; /** Dialog text used for analysis (if provided) */ dialogText?: string; }; } /** * Options for lip-sync-engine analysis */ interface LipSyncEngineOptions { /** * Optional dialog text for improved recognition accuracy * Providing the expected text significantly improves results */ dialogText?: string; /** * Sample rate of the audio buffer * @default 16000 * @recommended 16000 for best PocketSphinx results */ sampleRate?: number; } /** * Progress callback for analysis */ type ProgressCallback = (progress: number) => void; /** * WASM module interface (internal) */ interface LipSyncEngineModule { _malloc(size: number): number; _free(ptr: number): void; _lipsyncengine_init(modelsPathPtr: number): number; _lipsyncengine_analyze_pcm16(pcm16Ptr: number, sampleCount: number, sampleRate: number, dialogPtr: number): number; _lipsyncengine_free(ptr: number): void; _lipsyncengine_get_last_error(): number; _lipsyncengine_cleanup(): void; HEAP16: Int16Array; lengthBytesUTF8(str: string): number; stringToUTF8(str: string, ptr: number, maxLen: number): void; UTF8ToString(ptr: number): string; } /** * WASM loader options */ interface WasmLoaderOptions { /** Path to the .wasm file */ wasmPath?: string; /** Path to the .data file */ dataPath?: string; /** Path to the .js file */ jsPath?: string; } /** * Main API class for Lip Sync * Framework-agnostic - works with vanilla JS, React, Vue, Svelte, etc. * * @example Vanilla JavaScript * ```typescript * import { LipSyncEngine } from 'lip-sync-engine'; * * const lipSyncEngine = LipSyncEngine.getInstance(); * await lipSyncEngine.init(); * * const result = await lipSyncEngine.analyze(pcm16Buffer, { * dialogText: "Hello world", * sampleRate: 16000 * }); * * console.log(result.mouthCues); * ``` * * @example React * ```typescript * // Create a custom hook in your app * function useLipSyncEngine() { * const [result, setResult] = useState(null); * const lipSyncEngineRef = useRef(LipSyncEngine.getInstance()); * * useEffect(() => { * lipSyncEngineRef.current.init(); * return () => lipSyncEngineRef.current.destroy(); * }, []); * * const analyze = async (pcm16, options) => { * const result = await lipSyncEngineRef.current.analyze(pcm16, options); * setResult(result); * }; * * return { analyze, result }; * } * ``` */ declare class LipSyncEngine { private static instance; private module; private initialized; private initPromise; private constructor(); /** * Get singleton instance */ static getInstance(): LipSyncEngine; /** * Initialize the WASM module * Call this once before using analyze() * * @param options - Optional paths to WASM files */ init(options?: WasmLoaderOptions): Promise; /** * Analyze audio and generate lip-sync-engine data * * @param pcm16 - 16-bit PCM audio buffer (mono, 16kHz recommended) * @param options - Optional configuration * @returns Promise resolving to lip-sync-engine result with mouth cues * * @throws {TypeError} If pcm16 is not an Int16Array * @throws {Error} If audio buffer is empty * @throws {Error} If analysis fails */ analyze(pcm16: Int16Array, options?: LipSyncEngineOptions): Promise; /** * Analyze audio using Web Worker (non-blocking) * Recommended for long audio files to avoid blocking UI * * @param pcm16 - 16-bit PCM audio buffer * @param options - Optional configuration * @returns Promise resolving to lip-sync-engine result */ analyzeAsync(pcm16: Int16Array, options?: LipSyncEngineOptions): Promise; /** * Destroy the instance and free resources * Call this when you're completely done with lip-sync-engine analysis */ destroy(): void; } /** * Convenience function for one-off analysis * Framework-agnostic - works everywhere */ declare const analyze: (pcm16: Int16Array, options?: LipSyncEngineOptions) => Promise; /** * Convenience function for async analysis with Web Worker */ declare const analyzeAsync: (pcm16: Int16Array, options?: LipSyncEngineOptions) => Promise; /** * Loads the WASM module * This is a singleton loader that handles WASM initialization */ declare class WasmLoader { private static modulePromise; private static module; /** * Load the WASM module * @param options - Optional paths to WASM files * @returns Promise resolving to the loaded WASM module */ static load(options?: WasmLoaderOptions): Promise; /** * Load module directly (for worker context) * This method doesn't use singleton caching and is suitable for workers */ static loadModule(options?: WasmLoaderOptions): Promise; /** * Internal method to load the module */ private static _loadModule; /** * Implementation of module loading that works in both window and worker contexts */ private static _loadModuleImpl; /** * Reset the loader (useful for testing) */ static reset(): void; } /** * Web Worker pool for non-blocking lip-sync-engine analysis * Manages multiple workers with automatic load balancing */ declare class WorkerPool { private static instance; private workers; private queue; private inFlightJobs; private nextJobId; private maxWorkers; private workerScriptUrl; private wasmPaths; private initialized; private constructor(); /** * Get singleton instance */ static getInstance(maxWorkers?: number, workerScriptUrl?: string): WorkerPool; /** * Initialize the worker pool * Must be called before using analyze() * * Strategy: Start with 1 worker for fast initialization * - Use warmup() to create all workers upfront if needed * - Workers scale on-demand automatically */ init(options?: { wasmPath?: string; dataPath?: string; jsPath?: string; workerScriptUrl?: string; }): Promise; /** * Pre-create all workers up to maxWorkers * Call this during app initialization to eliminate worker creation overhead * * @example * ```typescript * const pool = WorkerPool.getInstance(); * await pool.init({ ... }); * await pool.warmup(); // Pre-create all workers * // Now all workers are ready for chunked processing * ``` */ warmup(): Promise; /** * Create a new worker and initialize it */ private createWorker; /** * Handle messages from workers */ private handleWorkerMessage; /** * Remove a worker from the pool */ private removeWorker; /** * Process queued jobs * Simple algorithm: assign each queued job to the next available idle worker * Workers must be pre-created via init() or warmup() */ private processQueue; /** * Assign a job to a worker */ private assignJobToWorker; /** * Analyze audio in a Web Worker (non-blocking) * * @param pcm16 - 16-bit PCM audio buffer * @param options - Optional configuration * @returns Promise resolving to lip-sync-engine result */ analyze(pcm16: Int16Array, options?: LipSyncEngineOptions): Promise; /** * Analyze multiple audio buffers in parallel using chunked processing * * @param chunks - Array of audio chunks to process * @param options - Optional configuration * @returns Promise resolving to array of results in same order as input */ analyzeChunks(chunks: Int16Array[], options?: LipSyncEngineOptions): Promise; /** * Create a dynamic streaming analyzer that allows adding chunks on-the-fly * Returns a controller that lets you add chunks as they arrive from a stream * * @param options - Optional configuration for all chunks * @returns StreamController with addChunk() and finalize() methods * * @example * ```typescript * const stream = pool.createStreamAnalyzer({ dialogText: "hello" }); * await pool.warmup(); // Pre-create workers * * // Add chunks as they arrive from your audio stream * for await (const chunk of audioStream) { * stream.addChunk(chunk); // Non-blocking, queues immediately * } * * // Wait for all chunks to complete and get results in order * const results = await stream.finalize(); * ``` */ createStreamAnalyzer(options?: LipSyncEngineOptions): StreamAnalyzerController; /** * Get pool statistics */ getStats(): { totalWorkers: number; busyWorkers: number; idleWorkers: number; queuedJobs: number; maxWorkers: number; }; /** * Destroy the worker pool and terminate all workers */ destroy(): void; } /** * Controller for dynamic streaming analysis * Allows adding chunks on-the-fly as they arrive from a stream */ declare class StreamAnalyzerController { private pool; private options; private chunks; private nextIndex; private finalized; constructor(pool: WorkerPool, options: LipSyncEngineOptions); /** * Add a chunk to be analyzed * This immediately queues the chunk for processing - no main thread blocking * * @param chunk - Audio chunk to analyze * @returns The index of this chunk in the result array */ addChunk(chunk: Int16Array): number; /** * Wait for all chunks to complete and return results in insertion order * * @returns Array of results in the same order chunks were added */ finalize(): Promise; /** * Get current streaming statistics */ getStats(): { chunksAdded: number; chunksCompleted: number; poolStats: ReturnType; }; } /** * Audio format conversion utilities * Framework-agnostic - works with any audio API */ /** * Convert Float32Array to Int16Array PCM */ declare function float32ToInt16(float32: Float32Array): Int16Array; /** * Convert AudioBuffer to Int16Array PCM * @param audioBuffer - Web Audio API AudioBuffer * @param targetSampleRate - Target sample rate (default: 16000) * @returns Int16Array PCM data */ declare function audioBufferToInt16(audioBuffer: AudioBuffer, targetSampleRate?: number): Int16Array; /** * Simple linear resampling * For better quality, consider using a dedicated library * * @param input - Input audio samples * @param fromRate - Source sample rate * @param toRate - Target sample rate * @returns Resampled audio */ declare function resample(input: Float32Array, fromRate: number, toRate: number): Float32Array; /** * Record audio from microphone * * @param durationMs - Recording duration in milliseconds * @param sampleRate - Target sample rate (default: 16000) * @returns Object containing PCM16 data and AudioBuffer */ declare function recordAudio(durationMs: number, sampleRate?: number): Promise<{ pcm16: Int16Array; audioBuffer: AudioBuffer; }>; /** * Load audio from a URL or File * * @param source - URL string or File object * @param targetSampleRate - Target sample rate (default: 16000) * @returns Object containing PCM16 data and AudioBuffer */ declare function loadAudio(source: string | File, targetSampleRate?: number): Promise<{ pcm16: Int16Array; audioBuffer: AudioBuffer; }>; /** * Web Worker entry point for lip-sync-engine * This file runs inside a Web Worker context and handles analysis requests */ interface WorkerAnalyzeRequest { type: 'analyze'; id: number; pcm16: Int16Array; options: LipSyncEngineOptions; } interface WorkerAnalyzeResponse { type: 'result' | 'error'; id: number; result?: LipSyncEngineResult; error?: string; } interface WorkerInitRequest { type: 'init'; wasmPath: string; dataPath: string; jsPath: string; } interface WorkerInitResponse { type: 'ready' | 'error'; error?: string; } type WorkerRequest = WorkerAnalyzeRequest | WorkerInitRequest; type WorkerResponse = WorkerAnalyzeResponse | WorkerInitResponse; export { LipSyncEngine, type LipSyncEngineModule, type LipSyncEngineOptions, type LipSyncEngineResult, type MouthCue, type ProgressCallback, StreamAnalyzerController, WasmLoader, type WasmLoaderOptions, type WorkerAnalyzeRequest, type WorkerAnalyzeResponse, type WorkerInitRequest, type WorkerInitResponse, WorkerPool, type WorkerRequest, type WorkerResponse, analyze, analyzeAsync, audioBufferToInt16, float32ToInt16, loadAudio, recordAudio, resample };