import type { ResolvedDevice } from "../../device.js"; import type { PcmAudio } from "../../platform.js"; import type { EmbedResult, SynthesisEngine, SynthesisRequest } from "../types.js"; import type { DtypeConfig, LoadPlan } from "./dtype-plan.js"; import type { LoadProgress, TransformersModuleLoader } from "./transformers-module.js"; /** * Sample rate of Chatterbox output. * * Fixed by the S3Gen vocoder: upstream `chatterbox/models/s3gen/const.py` * defines `S3GEN_SR = 24000`. */ export declare const CHATTERBOX_SAMPLE_RATE = 24000; /** * Engine-level load milestones, emitted on the same channel as the file * progress forwarded from Transformers.js. * * The `load-` prefix keeps them distinct from the library's own statuses * (`initiate` / `download` / `progress` / `progress_total` / `done` / `ready`) * and from the generic `ready` that {@link exposeEngine} emits for any engine. * * Note there is deliberately no "downloads finished / compiling now" event. * Detecting that transition needs a trustworthy count of the files a load will * touch, and Transformers.js cannot supply one for this model: `chatterbox` is * absent from its `MODEL_TYPE_MAPPING`, so its expected-file list resolves to a * single-file encoder-only layout that does not exist in the repo. Emitting a * milestone inferred from silence would just move the consumer's guess into the * library and dress it up as a fact. */ export interface ChatterboxLifecycleEvent { readonly status: "load-start" | "load-fallback" | "load-ready"; /** The plan this event concerns, as `device/dtype` — e.g. `webgpu/q4f16`. */ readonly plan: string; readonly device: ResolvedDevice; /** Quantization of the language model, the only session that varies. */ readonly dtype: string; /** Why the plan was abandoned. Only present on `load-fallback`. */ readonly reason?: string; } /** * Generation stopped because it reached its token budget rather than because * the utterance had finished, so the audio ends early. * * Carried on the same channel as the load milestones because that is the only * channel there is. A declared variant rather than something cast into place, * so consumers can narrow to it. */ export interface ChatterboxTruncationEvent { readonly status: "synthesize-truncated"; /** The chunk that did not fit. */ readonly text: string; /** The budget it ran out of. */ readonly tokens: number; } /** Everything {@link ChatterboxEngineOptions.onProgress} may receive. */ export type ChatterboxLoadEvent = LoadProgress | ChatterboxLifecycleEvent | ChatterboxTruncationEvent; export interface ChatterboxEngineOptions { /** * Hugging Face model id. * * @defaultValue "onnx-community/chatterbox-ONNX" */ modelId?: string; /** Override the per-session quantization chosen for the device. */ dtype?: Partial; /** * Fixed generation cap, honoured as written for every chunk. * * Leave it unset — the default — and the cap is sized to each chunk from its * length instead, so a full-length chunk is not truncated and a short one * does not pay for tokens it will never use. A fixed cap that is too small * for the text ends the audio mid-sentence; see {@link TOKENS_PER_CHARACTER} * for the measured relationship, and the `synthesize-truncated` event for * when it happens anyway. * * Must be a positive integer. It is honoured as written, including above the * ceiling that applies to a derived budget: naming a number is deliberate. */ maxNewTokens?: number; /** * Chatterbox's expressiveness control. * * @defaultValue 0.5 */ exaggeration?: number; /** * Called with model download progress forwarded from Transformers.js, and * with the engine's own {@link ChatterboxLifecycleEvent} milestones. * * Leaving it unset does not make the load cheaper. Transformers.js is always * given a `progress_callback`, because that is the only heartbeat * {@link stallTimeoutMs} has, and upstream gates a metadata probe per * expected file on its presence. Every load pays for that round of requests * whether or not anything is listening. */ onProgress?: (progress: ChatterboxLoadEvent) => void; /** * Reject {@link ChatterboxEngine.load} when it produces no progress for this * long. The clock is reset by every progress event, so it measures silence * rather than total elapsed time — a total cap would abandon healthy loads, * because a 1.5 GB download legitimately takes minutes. * * A hung transfer never rejects on its own, so without this `load()` stays * pending forever and the caller has no error to catch. Set `0` — or * `Infinity`, which means the same thing — to wait indefinitely. * * Anything else must be at least 1 millisecond. A smaller value is rejected * rather than rounded: `setTimeout` would round it up and fire before the * transfer could produce anything, and it is far more likely to be seconds * written where milliseconds were meant. * * @defaultValue 300000 */ stallTimeoutMs?: number; /** * How to obtain `@huggingface/transformers`. Replace it in tests, or to * pin your own build of the library. */ loadModule?: TransformersModuleLoader; /** * Whether the WebGPU adapter can run f16 shaders. Only consulted when the * resolved device is `webgpu`. Defaults to asking `navigator.gpu` for an * adapter and checking `features.has("shader-f16")` — an f16 model on a * device without it loads fine and then fails at the first inference, so * the check must happen before the load plans are built. */ supportsFp16?: () => boolean | Promise; } /** * Zero-shot TTS backed by Chatterbox ONNX through Transformers.js v4. * * The model is split into four ONNX sessions (`embed_tokens`, * `speech_encoder`, `language_model`, `conditional_decoder`). Cloning runs the * speech encoder once and keeps its four output tensors, so every later * `synthesize` call skips straight to generation. * * `@huggingface/transformers` is an optional peer dependency: it is imported * lazily on first `load()`, so applications that supply another engine never * pay for it. */ export declare class ChatterboxEngine implements SynthesisEngine { #private; readonly name = "chatterbox"; readonly sampleRate = 24000; readonly modelId: string; constructor(options?: ChatterboxEngineOptions); /** The device / dtype combination that actually loaded, once `load` ran. */ get loadedPlan(): LoadPlan | undefined; /** * Download (or read from the browser cache) and initialise the model. * * Each plan from {@link buildLoadPlans} is tried in turn, so a GPU without * fp16 support degrades instead of failing outright. */ load(device: ResolvedDevice): Promise; /** Run the speech encoder over reference audio and keep its speaker tensors. */ embed(audio: PcmAudio): Promise; /** Render one chunk of speech with the given voice. */ synthesize(request: SynthesisRequest): Promise; /** Free the ONNX sessions. A later `load()` re-creates them. */ dispose(): Promise; } //# sourceMappingURL=chatterbox-engine.d.ts.map