// Exode audio: timeline clips and one-shot sounds on Web Audio. // // Shared by the editor and every web runtime. Edit it here, then run // `node scripts/sync-audio.mjs` to copy it into the other runtimes. // // Performance rules this module keeps: // - Nothing runs for files without audio: no AudioContext and no listeners until an asset exists. // - Clips are scheduled on the audio clock. Each frame only reconciles what should be sounding // with what is sounding (O(clips), no allocations in steady state); a voice is (re)started only // on play, seek, loop wrap, edit, or drift beyond DRIFT_LIMIT. // - Short sounds are decoded once per source and shared across engines; long music streams // through a media element instead of holding a large decoded buffer in memory. // - Waveform peaks are computed once at import and stored in the file, so a timeline never decodes // audio just to draw it. export interface ExodeAudioAsset { id: string; name: string; /** data: URI, blob: URL, http(s) URL or bundle `asset://` key. */ src: string; /** Seconds. */ duration: number; mime?: string; /** base64 Uint8 waveform: max |sample| per bucket, 0–255. Editor-only. */ peaks?: string; peaksPerSecond?: number; /** Force a playback path. Default: stream when longer than `streamThreshold`. */ mode?: 'buffer' | 'stream'; /** Content hash (editor): re-importing the same file reuses the asset. */ hash?: string; /** Seconds of encoder padding at the head of `src` (compressed imports); skipped on playback. */ startPad?: number; /** Attribution for library sounds (all CC0, kept for provenance). */ credit?: string; } export interface ExodeAudioClip { id: string; assetId: string; name?: string; /** Timeline second where the clip starts. */ start: number; /** Seconds trimmed off the head of the source. */ offset?: number; /** Clip length on the timeline. Default: the rest of the source (or the timeline, if looping). */ duration?: number; /** 0–2 (1 = unity). */ volume?: number; /** -1 left … 1 right. */ pan?: number; /** Playback rate; also shifts pitch. */ rate?: number; fadeIn?: number; fadeOut?: number; /** Repeat the trimmed source until the clip ends. */ loop?: boolean; muted?: boolean; solo?: boolean; /** Editor lane (row) index. */ lane?: number; /** Volume automation: clip-relative seconds → gain multiplier 0–2, linear between points. */ volumeEnvelope?: { time: number; value: number }[]; } export interface ExodeAudioConfig { assets: ExodeAudioAsset[]; /** Master volume 0–2. */ volume?: number; muted?: boolean; /** Sources longer than this (seconds) stream. Default 30. */ streamThreshold?: number; } export interface ExodeSoundOptions { volume?: number; pan?: number; rate?: number; /** Random ±semitones per play, so repeated hits don't sound mechanical. */ pitchVariance?: number; loop?: boolean; /** Seconds into the source. */ offset?: number; /** 'overlap' layers voices (default), 'restart' cuts the previous one, 'ignore' skips while playing. */ mode?: 'overlap' | 'restart' | 'ignore'; /** Cap on simultaneous voices of this asset in overlap mode. Default 8. */ maxVoices?: number; } export interface TimelineSyncOptions { playing: boolean; /** Engine playback rate (state speed). Negative plays silent. */ rate?: number; /** Timeline length; clips are cut here. */ duration?: number; } /** * Sub-animation id for an animation's audio clips. A play with a sub-animation filter (flow rule * or Play Animation with selected objects) sounds only when the filter includes it. */ export const AUDIO_SUB_ANIMATION_ID = '__audio__'; /** Whether a layer playing with `filterObjectIds` should play its animation's clips. */ export function filterPlaysAudio(filterObjectIds: string[] | null | undefined): boolean { return !filterObjectIds || filterObjectIds.includes(AUDIO_SUB_ANIMATION_ID); } /** One sound as it lands in a rendered video (output seconds). Produced by the recorder or from clips. */ export interface AudioRenderEvent { src: string; /** Output time the sound starts. */ at: number; /** Source position at `at`. */ offset: number; /** Output seconds it plays for (Infinity = until the render ends). */ dur: number; rate: number; volume: number; pan: number; loop?: { start: number; end?: number }; /** Gain multiplier automation (fades, envelope) as [output time, gain]; linear between points. */ gain?: [number, number][]; } /** * Recorder mode (headless video renders): voices aren't played, they're logged with the render * clock, then mixed offline with `mixAudioEvents`. Whatever the engine would play — timeline * clips through states, Play Sound from scripted clicks — lands in the video. */ let recorder: { now(): number; events: AudioRenderEvent[] } | null = null; export function setAudioRecorder(r: { now(): number; events: AudioRenderEvent[] } | null) { recorder = r; } /** Render events into one AudioBuffer with OfflineAudioContext (faster than real time, sample-exact). */ export async function mixAudioEvents(events: AudioRenderEvent[], duration: number, opts: { sampleRate?: number; volume?: number; startAt?: number } = {}): Promise { const sr = opts.sampleRate ?? 48000; const frames = Math.max(1, Math.ceil(duration * sr)); const OAC = (globalThis as any).OfflineAudioContext || (globalThis as any).webkitOfflineAudioContext; if (!OAC || !events.length) return null; const ctx: OfflineAudioContext = new OAC(2, frames, sr); const shift = opts.startAt ?? 0; const master = ctx.createGain(); master.gain.value = opts.volume ?? 1; master.connect(ctx.destination); const decoded = new Map>(); const decode = (src: string) => { if (!decoded.has(src)) decoded.set(src, fetch(src).then(r => r.arrayBuffer()).then(b => ctx.decodeAudioData(b)).catch(() => null)); return decoded.get(src)!; }; await Promise.all(events.map(async ev => { const buffer = await decode(ev.src); if (!buffer) return; let at = ev.at - shift, offset = ev.offset, dur = Math.min(ev.dur, duration + shift - ev.at); if (at < 0) { offset += -at * ev.rate; dur += at; at = 0; } // started before the output window if (dur <= 0.001 || at >= duration) return; if (!ev.loop && offset >= buffer.duration) return; const src = ctx.createBufferSource(); src.buffer = buffer; src.playbackRate.value = ev.rate; if (ev.loop) { src.loop = true; src.loopStart = Math.min(buffer.duration, ev.loop.start); src.loopEnd = Math.min(buffer.duration, ev.loop.end ?? buffer.duration); } const g = ctx.createGain(); g.gain.setValueAtTime(0, at); const g0 = ev.gain?.length ? interpolatePoints(ev.gain, ev.at) : 1; g.gain.linearRampToValueAtTime(g0 * ev.volume, at + DECLICK); for (const [t, v] of ev.gain || []) { const ot = t - shift; if (ot > at + DECLICK && ot < at + dur) g.gain.linearRampToValueAtTime(v * ev.volume, ot); } const end = at + dur; g.gain.setValueAtTime(interpolatePoints(ev.gain || [], end + shift, 1) * ev.volume, Math.max(at + DECLICK, end - DECLICK)); g.gain.linearRampToValueAtTime(0, end); let out: AudioNode = g; if (ev.pan && typeof (ctx as any).createStereoPanner === 'function') { const p = ctx.createStereoPanner(); p.pan.value = Math.max(-1, Math.min(1, ev.pan)); g.connect(p); out = p; } src.connect(g); out.connect(master); src.start(at, Math.max(0, Math.min(buffer.duration - 0.0005, offset))); src.stop(end + 0.002); })); return ctx.startRendering(); } function interpolatePoints(pts: [number, number][], t: number, fallback = 1): number { if (!pts.length) return fallback; if (t <= pts[0][0]) return pts[0][1]; for (let i = 1; i < pts.length; i++) { if (t <= pts[i][0]) { const [t0, v0] = pts[i - 1], [t1, v1] = pts[i]; return t1 === t0 ? v1 : v0 + (v1 - v0) * (t - t0) / (t1 - t0); } } return pts[pts.length - 1][1]; } /** Events for a timeline's clips played from 0 (browser export; no recorder needed). */ export function eventsFromClips(clips: ExodeAudioClip[] | undefined, assets: ExodeAudioAsset[], duration: number): AudioRenderEvent[] { const byId = new Map(assets.map(a => [a.id, a])); const anySolo = (clips || []).some(c => c.solo); const out: AudioRenderEvent[] = []; for (const clip of clips || []) { const asset = byId.get(clip.assetId); if (!asset || clip.muted || (anySolo && !clip.solo)) continue; const len = clipLength(clip, asset, duration); if (len <= 0.001) continue; out.push({ src: asset.src, at: clip.start, offset: (asset.startPad || 0) + (clip.offset || 0), dur: len, rate: clip.rate && clip.rate > 0 ? clip.rate : 1, volume: clip.volume ?? 1, pan: clip.pan ?? 0, loop: clip.loop ? { start: (asset.startPad || 0) + (clip.offset || 0), end: (asset.startPad || 0) + asset.duration } : undefined, gain: gainPoints(clip, len, 0, len, clip.start, 1), }); } return out; } /** Fade/envelope automation for clip time [from, len) as output-time points. */ function gainPoints(clip: ExodeAudioClip, len: number, from: number, _to: number, outAtFrom: number, engineRate: number): [number, number][] | undefined { if (!clip.fadeIn && !clip.fadeOut && !clip.volumeEnvelope?.length) return undefined; const ts = new Set([from, len]); if (clip.fadeIn) ts.add(clip.fadeIn); if (clip.fadeOut) ts.add(Math.max(0, len - clip.fadeOut)); (clip.volumeEnvelope || []).forEach(p => ts.add(p.time)); return [...ts].filter(t => t >= from && t <= len).sort((a, b) => a - b) .map(t => [outAtFrom + (t - from) / engineRate, clipGainAt(clip, Math.min(t, len - 1e-4), len)]); } /** 16-bit PCM WAV bytes for an AudioBuffer. */ export function encodeWav(buffer: AudioBuffer): Uint8Array { const ch = buffer.numberOfChannels, n = buffer.length, sr = buffer.sampleRate; const out = new DataView(new ArrayBuffer(44 + n * ch * 2)); const str = (o: number, t: string) => { for (let i = 0; i < t.length; i++) out.setUint8(o + i, t.charCodeAt(i)); }; str(0, 'RIFF'); out.setUint32(4, 36 + n * ch * 2, true); str(8, 'WAVE'); str(12, 'fmt '); out.setUint32(16, 16, true); out.setUint16(20, 1, true); out.setUint16(22, ch, true); out.setUint32(24, sr, true); out.setUint32(28, sr * ch * 2, true); out.setUint16(32, ch * 2, true); out.setUint16(34, 16, true); str(36, 'data'); out.setUint32(40, n * ch * 2, true); const data: Float32Array[] = []; for (let c = 0; c < ch; c++) data.push(buffer.getChannelData(c)); let o = 44; for (let i = 0; i < n; i++) for (let c = 0; c < ch; c++) { const v = Math.max(-1, Math.min(1, data[c][i])); out.setInt16(o, v < 0 ? v * 0x8000 : v * 0x7fff, true); o += 2; } return new Uint8Array(out.buffer); } const DRIFT_LIMIT = 0.08; // seconds of audio-vs-visual drift before a voice is re-synced const JUMP_LIMIT = 0.12; // a time change this far off the expected step is a seek const LOOKAHEAD = 0.2; // schedule clips that start within this window const DECLICK = 0.006; // micro-fade on every start/stop const WATCHDOG_MS = 250; // timelines that stop being synced (paused RAF, hidden tab) go silent const BUFFER_CACHE_LIMIT = 192 * 1024 * 1024; // bytes of decoded PCM kept across engines // --------------------------------------------------------------------------------------------- // Shared context, unlock, and decode cache (one per page, shared by all engines and the editor). // --------------------------------------------------------------------------------------------- let sharedCtx: AudioContext | null = null; let unlockInstalled = false; function hasWebAudio(): boolean { return typeof window !== 'undefined' && !!((window as any).AudioContext || (window as any).webkitAudioContext); } export function getAudioContext(): AudioContext | null { if (sharedCtx) return sharedCtx; if (!hasWebAudio()) return null; const Ctor = (window as any).AudioContext || (window as any).webkitAudioContext; try { sharedCtx = new Ctor({ latencyHint: 'interactive' }) as AudioContext; } catch { return null; } installUnlock(); return sharedCtx; } /** Browsers start contexts suspended until a user gesture; resume on the first one. */ function installUnlock() { if (unlockInstalled || typeof window === 'undefined') return; unlockInstalled = true; const events = ['pointerdown', 'keydown', 'touchend', 'click']; const unlock = () => { const ctx = getAudioContext(); if (!ctx) return; if (ctx.state === 'suspended') ctx.resume().catch(() => { /* retried on next gesture */ }); if (ctx.state === 'running') events.forEach(e => window.removeEventListener(e, unlock, true)); }; events.forEach(e => window.addEventListener(e, unlock, true)); } if (typeof window !== 'undefined') { installUnlock(); } /** True once the browser lets audio play (after a gesture). */ export function isAudioUnlocked(): boolean { return !!sharedCtx && sharedCtx.state === 'running'; } /** Call from a click handler (e.g. a Play button) to unlock audio explicitly. */ export function unlockAudio(): Promise { const ctx = getAudioContext(); if (!ctx || ctx.state === 'running') return Promise.resolve(); return ctx.resume().catch(() => undefined); } interface CachedBuffer { buffer: AudioBuffer | null; promise: Promise; bytes: number; used: number; } const bufferCache = new Map(); let cachedBytes = 0; function evictBuffers() { if (cachedBytes <= BUFFER_CACHE_LIMIT) return; const entries = [...bufferCache.entries()].filter(([, c]) => c.buffer).sort((a, b) => a[1].used - b[1].used); for (const [key, c] of entries) { if (cachedBytes <= BUFFER_CACHE_LIMIT * 0.75) break; bufferCache.delete(key); cachedBytes -= c.bytes; } } /** Decode a source once; concurrent callers share the same promise. */ export function loadAudioBuffer(src: string): Promise { const hit = bufferCache.get(src); if (hit) { hit.used = performance.now(); return hit.promise; } const ctx = getAudioContext(); if (!ctx) return Promise.resolve(null); const entry: CachedBuffer = { buffer: null, promise: null as any, bytes: 0, used: performance.now() }; entry.promise = fetch(src) .then(r => r.arrayBuffer()) .then(data => ctx.decodeAudioData(data)) .then(buffer => { entry.buffer = buffer; entry.bytes = buffer.length * buffer.numberOfChannels * 4; cachedBytes += entry.bytes; evictBuffers(); return buffer; }) .catch(err => { bufferCache.delete(src); if (typeof console !== 'undefined') console.warn('[ExodeAudio] could not decode', src.slice(0, 64), err); return null; }); bufferCache.set(src, entry); return entry.promise; } const warmQueue: string[] = []; let warmActive = 0; /** Background decode: low priority, two at a time, in idle time. On-demand loads skip the queue. */ export function warmAudioBuffer(src: string) { if (bufferCache.has(src) || warmQueue.includes(src)) return; warmQueue.push(src); pumpWarm(); } function pumpWarm() { while (warmActive < 2 && warmQueue.length) { const src = warmQueue.shift()!; if (bufferCache.has(src)) continue; warmActive++; const run = () => loadAudioBuffer(src).finally(() => { warmActive--; pumpWarm(); }); const ric = typeof window !== 'undefined' ? (window as any).requestIdleCallback : undefined; if (ric) ric(run, { timeout: 500 }); else setTimeout(run, 16); } } /** Loop end inside the decoded buffer: the real audio, not the encoder's padded tail. */ function loopEndOf(asset: ExodeAudioAsset, buffer: AudioBuffer): number { return asset.startPad ? Math.min(buffer.duration, asset.startPad + asset.duration) : buffer.duration; } function cachedBuffer(src: string): AudioBuffer | null { const hit = bufferCache.get(src); if (!hit) return null; hit.used = performance.now(); return hit.buffer; } // --------------------------------------------------------------------------------------------- // Waveform peaks (editor import, MCP, tooling). // --------------------------------------------------------------------------------------------- /** Max |sample| per bucket across channels, quantised to 0–255. */ export function computePeaks(buffer: AudioBuffer, peaksPerSecond = 100): Uint8Array { const bucket = Math.max(1, Math.floor(buffer.sampleRate / peaksPerSecond)); const count = Math.ceil(buffer.length / bucket); const out = new Uint8Array(count); const channels: Float32Array[] = []; for (let c = 0; c < buffer.numberOfChannels; c++) channels.push(buffer.getChannelData(c)); for (let i = 0; i < count; i++) { const from = i * bucket; const to = Math.min(buffer.length, from + bucket); let max = 0; for (const data of channels) { for (let s = from; s < to; s++) { const v = data[s] < 0 ? -data[s] : data[s]; if (v > max) max = v; } } out[i] = Math.min(255, Math.round(max * 255)); } return out; } export function encodePeaks(peaks: Uint8Array): string { let s = ''; for (let i = 0; i < peaks.length; i += 0x8000) s += String.fromCharCode.apply(null, peaks.subarray(i, i + 0x8000) as any); return btoa(s); } const peakDecodeCache = new Map(); export function decodePeaks(b64: string | undefined): Uint8Array | null { if (!b64) return null; const hit = peakDecodeCache.get(b64); if (hit) return hit; const bin = atob(b64); const out = new Uint8Array(bin.length); for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); if (peakDecodeCache.size > 64) peakDecodeCache.clear(); peakDecodeCache.set(b64, out); return out; } // --------------------------------------------------------------------------------------------- // Clip timing helpers (pure; used by engine, editor timeline and tests). // --------------------------------------------------------------------------------------------- /** Clip length on the timeline, cut at `timelineDuration` when given. */ export function clipLength(clip: ExodeAudioClip, asset: ExodeAudioAsset | undefined, timelineDuration?: number): number { const rate = clip.rate && clip.rate > 0 ? clip.rate : 1; let len: number; if (typeof clip.duration === 'number' && clip.duration > 0) len = clip.duration; else if (clip.loop) len = Math.max(0, (timelineDuration ?? Infinity) - clip.start); else len = Math.max(0, ((asset?.duration ?? 0) - (clip.offset || 0)) / rate); if (typeof timelineDuration === 'number' && isFinite(timelineDuration)) len = Math.min(len, Math.max(0, timelineDuration - clip.start)); return len; } /** Gain multiplier from fades + envelope at clip-relative time `t`. */ export function clipGainAt(clip: ExodeAudioClip, t: number, length: number): number { let g = 1; const fi = clip.fadeIn || 0, fo = clip.fadeOut || 0; if (fi > 0 && t < fi) g *= Math.max(0, t / fi); if (fo > 0 && t > length - fo) g *= Math.max(0, (length - t) / fo); const env = clip.volumeEnvelope; if (env && env.length) g *= envelopeAt(env, t); return g; } export function envelopeAt(env: { time: number; value: number }[], t: number): number { if (t <= env[0].time) return env[0].value; for (let i = 1; i < env.length; i++) { if (t <= env[i].time) { const a = env[i - 1], b = env[i]; const k = b.time === a.time ? 1 : (t - a.time) / (b.time - a.time); return a.value + (b.value - a.value) * k; } } return env[env.length - 1].value; } /** Source position (seconds) for clip-relative time `t`, honouring offset, rate and loop. */ export function sourcePosition(clip: ExodeAudioClip, asset: ExodeAudioAsset | undefined, t: number): number { const rate = clip.rate && clip.rate > 0 ? clip.rate : 1; const offset = clip.offset || 0; let pos = t * rate; if (clip.loop && asset && asset.duration - offset > 0.01) pos %= (asset.duration - offset); return (asset?.startPad || 0) + offset + pos; } // --------------------------------------------------------------------------------------------- // Voices. // --------------------------------------------------------------------------------------------- interface Voice { kind: 'buffer' | 'stream' | 'record'; /** Recorder mode: the logged event (its duration is cut when the voice stops). */ event?: AudioRenderEvent; clipId: string; /** Signature of the timing fields; a change restarts the voice. */ sig: string; /** Audio-clock time at which clip-relative t = 0. */ ctxAtClipZero: number; /** Engine (state speed) rate: clip-relative seconds advance this fast per audio second. */ engineRate: number; source?: AudioBufferSourceNode; el?: HTMLAudioElement; elNode?: MediaElementAudioSourceNode; fade: GainNode; vol: GainNode; pan: StereoPannerNode | null; lastVol: number; lastPan: number; lastMuted: boolean; ended: boolean; } interface TimelineState { voices: Map; lastTime: number; lastWall: number; synced: boolean; /** Timeline time ↔ audio clock, set when voices start; null while nothing is sounding. */ anchor: { time: number; ctx: number; rate: number } | null; } interface IndexedClip { clip: ExodeAudioClip; asset: ExodeAudioAsset; start: number; end: number; len: number; baseRate: number; muted: boolean; sig: string; } interface ClipIndex { owner: ExodeAudio; assetsVersion: number; duration: number | undefined; items: IndexedClip[]; /** Running max of `end` over items sorted by start (monotonic, so it can be binary-searched). */ maxEnd: Float64Array; byId: Map; } const clipIndexCache = new WeakMap(); interface OneShot { assetId: string; source?: AudioBufferSourceNode; el?: HTMLAudioElement; gain: GainNode; done: boolean; event?: AudioRenderEvent; } function clipSig(c: ExodeAudioClip, rate: number, len: number): string { return `${c.assetId}|${c.start}|${c.offset || 0}|${len}|${rate}|${c.loop ? 1 : 0}|${c.pan ? 1 : 0}|${c.fadeIn || 0}|${c.fadeOut || 0}|${c.volumeEnvelope ? c.volumeEnvelope.length + ':' + c.volumeEnvelope.map(p => p.time + ',' + p.value).join(';') : ''}`; } /** Stream elements are expensive to create; keep a small idle pool per source. */ const streamPool = new Map(); function takeStream(ctx: AudioContext, src: string) { const pool = streamPool.get(src); if (pool && pool.length) return pool.pop()!; const el = new Audio(); el.crossOrigin = 'anonymous'; el.preload = 'auto'; el.src = src; const node = ctx.createMediaElementSource(el); return { el, node }; } function returnStream(src: string, item: { el: HTMLAudioElement; node: MediaElementAudioSourceNode }) { try { item.el.pause(); item.node.disconnect(); } catch { /* already detached */ } const pool = streamPool.get(src) || []; if (pool.length < 3) { pool.push(item); streamPool.set(src, pool); } else { item.el.removeAttribute('src'); item.el.load(); } } // --------------------------------------------------------------------------------------------- // ExodeAudio: one per engine instance (and one for the editor timeline). // --------------------------------------------------------------------------------------------- export class ExodeAudio { private assets = new Map(); /** Bumped whenever assets change, so cached clip indexes rebuild. */ private assetsVersion = 0; private assetsByName = new Map(); private config: ExodeAudioConfig | null = null; private master: GainNode | null = null; private timelines = new Map(); private oneShots: OneShot[] = []; private frameTouched = new Set(); private watchdog: ReturnType | null = null; private lastScrubWall = 0; private visibilityHandler: (() => void) | null = null; private disposed = false; /** Host-app controls (page mute button, volume prop); survive file reloads and multiply the file's own master. */ private host = { volume: 1, muted: false }; /** True when the loaded file has at least one asset. Everything else is a no-op otherwise. */ get enabled(): boolean { return this.assets.size > 0 && !this.disposed; } load(config: ExodeAudioConfig | undefined | null, resolveSrc?: (src: string) => string) { this.stopAll(); this.assets.clear(); this.assetsByName.clear(); this.assetsVersion++; this.config = config && Array.isArray(config.assets) ? config : null; if (!this.config || !this.config.assets.length) return; for (const raw of this.config.assets) { if (!raw || !raw.id || !raw.src) continue; const asset = resolveSrc ? { ...raw, src: resolveSrc(raw.src) } : raw; this.assets.set(asset.id, asset); if (asset.name) this.assetsByName.set(asset.name, asset); } if (!this.assets.size) return; if (recorder) return; // nothing to play live; voices are logged if (!getAudioContext()) return; this.ensureMaster(); // Warm short sounds in the background (2 at a time, idle time) so the first trigger is // instant without a decode spike on load; long ones stream on demand. for (const asset of this.assets.values()) { if (!this.isStream(asset)) warmAudioBuffer(asset.src); } this.installVisibility(); } /** Update one asset's source/metadata (editor replace) without reloading. */ upsertAsset(asset: ExodeAudioAsset) { this.assets.set(asset.id, asset); this.assetsVersion++; if (asset.name) this.assetsByName.set(asset.name, asset); if (!this.config) this.config = { assets: [] }; if (getAudioContext()) { this.ensureMaster(); if (!this.isStream(asset)) warmAudioBuffer(asset.src); this.installVisibility(); } } getAsset(idOrName: string): ExodeAudioAsset | undefined { return this.assets.get(idOrName) || this.assetsByName.get(idOrName); } setMasterVolume(volume: number) { if (!this.config) this.config = { assets: [] }; this.config.volume = volume; this.applyMaster(); } setMuted(muted: boolean) { if (!this.config) this.config = { assets: [] }; this.config.muted = muted; this.applyMaster(); } get muted(): boolean { return !!this.config?.muted; } setHostVolume(volume: number) { this.host.volume = Math.max(0, volume); this.applyMaster(); } setHostMuted(muted: boolean) { this.host.muted = muted; this.applyMaster(); } get hostMuted(): boolean { return this.host.muted; } // -- per-frame timeline reconcile ---------------------------------------------------------- /** Mark the start of an engine frame; timelines not synced before endFrame() go silent. */ beginFrame() { this.frameTouched.clear(); } endFrame() { if (!this.timelines.size) return; for (const key of this.timelines.keys()) { if (!this.frameTouched.has(key)) this.stopTimeline(key); } } /** * Make what is sounding for `key` match `clips` at timeline `time`. * Call every frame while the timeline is on screen; cheap when nothing changes. */ syncTimeline(key: string, clips: ExodeAudioClip[] | undefined, time: number, opts: TimelineSyncOptions) { if (!clips || !clips.length || !this.enabled) { if (this.timelines.has(key)) this.stopTimeline(key); return; } this.frameTouched.add(key); const ctx = recorder ? null : getAudioContext(); if (!recorder && (!ctx || !this.master)) return; const engineRate = opts.rate ?? 1; if (!opts.playing || engineRate <= 0) { if (this.timelines.has(key)) this.stopTimeline(key); return; } const wall = performance.now(); let tl = this.timelines.get(key); if (!tl) { tl = { voices: new Map(), lastTime: time, lastWall: wall, synced: false, anchor: null }; this.timelines.set(key, tl); } else { // A seek, loop wrap or long stall: drop everything and re-enter at `time`. const expected = tl.lastTime + ((wall - tl.lastWall) / 1000) * engineRate; if (time < tl.lastTime - 1e-4 || Math.abs(time - expected) > JUMP_LIMIT) this.silenceVoices(tl); } tl.lastTime = time; tl.lastWall = wall; this.startWatchdog(); const index = this.indexFor(clips, opts.duration); const now = recorder ? recorder.now() : ctx!.currentTime; // 1. Stop voices whose clip ended, was deleted, muted, or sits outside the window. if (tl.voices.size) { for (const [id, v] of tl.voices) { const it = index.byId.get(id); if (!it || it.muted || time >= it.end || time < it.start - LOOKAHEAD) { this.stopVoice(v, now); tl.voices.delete(id); } } } // 2. Only clips around the playhead: binary search on the running max end, then walk // forward until clips start after the lookahead. O(log n + clips at the playhead). const items = index.items; let lo = 0, hi = items.length; while (lo < hi) { const mid = (lo + hi) >> 1; if (index.maxEnd[mid] > time) hi = mid; else lo = mid + 1; } const horizon = time + LOOKAHEAD; for (let i = lo; i < items.length; i++) { const it = items[i]; if (it.start > horizon) break; if (it.muted || time >= it.end) continue; // muted clips cost nothing; unmuting re-enters at the right spot const clip = it.clip; let voice = tl.voices.get(clip.id); if (voice && (voice.sig !== it.sig || voice.ended || voice.engineRate !== engineRate)) { this.stopVoice(voice, now); tl.voices.delete(clip.id); voice = undefined; } const rate = it.baseRate * engineRate; if (!voice) { voice = this.startClipVoice(ctx!, clip, it.asset, time, it.len, rate, it.sig) || undefined; if (!voice) continue; // still decoding; retried next frame tl.voices.set(clip.id, voice); } else { // Drift: compare the voice's clip position with the engine's. const visualT = time - clip.start; if (voice.kind === 'stream') { // Media elements start late and drift; nudge the element instead of restarting. const el = voice.el!; if (visualT > 0.25 && !el.seeking && el.readyState >= 2) { const expected = sourcePosition(clip, it.asset, visualT); if (Math.abs(el.currentTime - expected) > DRIFT_LIMIT * 2) { try { el.currentTime = expected; } catch { /* not seekable */ } } } } else if (visualT > 0.05 && Math.abs((now - voice.ctxAtClipZero) * voice.engineRate - visualT) > DRIFT_LIMIT) { this.stopVoice(voice, now); const fresh = this.startClipVoice(ctx!, clip, it.asset, time, it.len, rate, it.sig); if (fresh) tl.voices.set(clip.id, fresh); else tl.voices.delete(clip.id); voice = fresh || undefined; if (!voice) continue; } } this.applyVoiceMix(voice, clip.volume ?? 1, clip.pan ?? 0, false, now); } if (!tl.voices.size) tl.anchor = null; else if (!tl.anchor || tl.anchor.rate !== engineRate) tl.anchor = { time, ctx: now, rate: engineRate }; } /** Sorted clip table for one clips array (cached by array identity; clips are immutable). */ private indexFor(clips: ExodeAudioClip[], duration: number | undefined): ClipIndex { const cached = clipIndexCache.get(clips); if (cached && cached.owner === this && cached.assetsVersion === this.assetsVersion && cached.duration === duration) return cached; let anySolo = false; for (const c of clips) if (c.solo) { anySolo = true; break; } const items: IndexedClip[] = []; for (const clip of clips) { const asset = this.assets.get(clip.assetId); if (!asset) continue; const len = clipLength(clip, asset, duration); if (len <= 0.001) continue; const baseRate = clip.rate && clip.rate > 0 ? clip.rate : 1; items.push({ clip, asset, len, baseRate, start: clip.start, end: clip.start + len, muted: !!clip.muted || (anySolo && !clip.solo), sig: clipSig(clip, baseRate, len), }); } items.sort((a, b) => a.start - b.start); const maxEnd = new Float64Array(items.length); let m = -Infinity; items.forEach((it, i) => { m = Math.max(m, it.end); maxEnd[i] = m; }); const index: ClipIndex = { owner: this, assetsVersion: this.assetsVersion, duration, items, maxEnd, byId: new Map(items.map(it => [it.clip.id, it])) }; clipIndexCache.set(clips, index); return index; } /** * Audio is the master clock. While a timeline has sound playing, the engine should draw at * the time returned here instead of its own accumulated frame time: dropped or slow frames * then skip visuals instead of making the audio stutter. Returns null when there is nothing * to follow, or when `time` is far off (a seek or loop wrap, which the next sync handles). */ clockTime(key: string, time: number, duration?: number): number | null { const tl = this.timelines.get(key); const ctx = sharedCtx; if (recorder || !tl || !tl.anchor || !ctx || ctx.state !== 'running') return null; const t = tl.anchor.time + (ctx.currentTime - tl.anchor.ctx) * tl.anchor.rate; if (Math.abs(t - time) > 0.5) return null; if (duration !== undefined && t >= duration) return null; return t; } stopTimeline(key: string) { const tl = this.timelines.get(key); if (!tl) return; this.silenceVoices(tl); this.timelines.delete(key); } /** Short preview grain at `time` while scrubbing (throttled). */ scrub(clips: ExodeAudioClip[] | undefined, time: number, grain = 0.07) { if (!clips || !clips.length || !this.enabled) return; const ctx = getAudioContext(); if (!ctx || !this.master || ctx.state !== 'running') return; const wall = performance.now(); if (wall - this.lastScrubWall < 45) return; this.lastScrubWall = wall; const anySolo = clips.some(c => c.solo); for (const clip of clips) { if (clip.muted || (anySolo && !clip.solo)) continue; const asset = this.assets.get(clip.assetId); if (!asset) continue; const len = clipLength(clip, asset); const t = time - clip.start; if (t < 0 || t >= len) continue; const buffer = cachedBuffer(asset.src); if (!buffer) { loadAudioBuffer(asset.src); continue; } const src = ctx.createBufferSource(); src.buffer = buffer; const g = ctx.createGain(); const level = (clip.volume ?? 1) * clipGainAt(clip, t, len) * 0.8; const now = ctx.currentTime; g.gain.setValueAtTime(0, now); g.gain.linearRampToValueAtTime(level, now + 0.01); g.gain.setValueAtTime(level, now + grain - 0.015); g.gain.linearRampToValueAtTime(0, now + grain); src.connect(g).connect(this.master); src.start(now, Math.min(buffer.duration - 0.01, sourcePosition(clip, asset, t)), grain + 0.01); } } // -- one-shot sounds (interactions, API) -------------------------------------------------- playSound(idOrName: string, options: ExodeSoundOptions = {}): boolean { if (recorder) return this.recordSound(idOrName, options); const asset = this.getAsset(idOrName); const ctx = getAudioContext(); if (!asset || !ctx || this.disposed) return false; this.ensureMaster(); if (!this.master) return false; if (ctx.state === 'suspended') ctx.resume().catch(() => undefined); this.reapOneShots(); const mode = options.mode || 'overlap'; const playing = this.oneShots.filter(s => s.assetId === asset.id && !s.done); if (mode === 'ignore' && playing.length) return false; if (mode === 'restart') playing.forEach(s => this.stopOneShot(s, ctx.currentTime)); if (mode === 'overlap' && playing.length >= (options.maxVoices ?? 8)) this.stopOneShot(playing[0], ctx.currentTime); let rate = options.rate && options.rate > 0 ? options.rate : 1; if (options.pitchVariance) rate *= Math.pow(2, ((Math.random() * 2 - 1) * options.pitchVariance) / 12); const gain = ctx.createGain(); const now = ctx.currentTime; const level = Math.max(0, options.volume ?? 1); gain.gain.setValueAtTime(0, now); gain.gain.linearRampToValueAtTime(level, now + DECLICK); let out: AudioNode = gain; if (options.pan) { const p = this.makePanner(ctx, options.pan); if (p) { gain.connect(p); out = p; } } out.connect(this.master); const shot: OneShot = { assetId: asset.id, gain, done: false }; if (this.isStream(asset)) { const item = takeStream(ctx, asset.src); item.node.connect(gain); item.el.loop = !!options.loop; item.el.playbackRate = rate; try { item.el.currentTime = (asset.startPad || 0) + (options.offset || 0); } catch { /* not seekable yet */ } item.el.play().catch(() => undefined); item.el.onended = () => { shot.done = true; returnStream(asset.src, item); }; shot.el = item.el; (shot as any).stream = item; } else { const buffer = cachedBuffer(asset.src); if (!buffer) { // First trigger before decode finished: play as soon as it lands (if still relevant). const requested = performance.now(); loadAudioBuffer(asset.src).then(b => { if (b && performance.now() - requested < 600 && !this.disposed) this.playSound(idOrName, options); }); gain.disconnect(); return false; } const src = ctx.createBufferSource(); src.buffer = buffer; src.loop = !!options.loop; if (src.loop) { src.loopStart = asset.startPad || 0; src.loopEnd = loopEndOf(asset, buffer); } src.playbackRate.value = rate; src.connect(gain); src.onended = () => { shot.done = true; try { gain.disconnect(); } catch { /* gone */ } }; src.start(now, Math.max(0, Math.min(buffer.duration - 0.001, (asset.startPad || 0) + (options.offset || 0)))); shot.source = src; } this.oneShots.push(shot); return true; } private recordSound(idOrName: string, options: ExodeSoundOptions): boolean { const asset = this.getAsset(idOrName); if (!asset || this.disposed) return false; const now = recorder!.now(); const mode = options.mode || 'overlap'; const playing = this.oneShots.filter(s => s.assetId === asset.id && !s.done && s.event && s.event.at + s.event.dur > now); if (mode === 'ignore' && playing.length) return false; if (mode === 'restart') playing.forEach(s => this.stopOneShot(s, now)); let rate = options.rate && options.rate > 0 ? options.rate : 1; if (options.pitchVariance) rate *= Math.pow(2, ((Math.random() * 2 - 1) * options.pitchVariance) / 12); const event: AudioRenderEvent = { src: asset.src, at: now, offset: (asset.startPad || 0) + (options.offset || 0), dur: options.loop ? Infinity : Math.max(0, asset.duration - (options.offset || 0)) / rate, rate, volume: Math.max(0, options.volume ?? 1), pan: options.pan ?? 0, loop: options.loop ? { start: asset.startPad || 0, end: (asset.startPad || 0) + asset.duration } : undefined, }; recorder!.events.push(event); this.oneShots.push({ assetId: asset.id, gain: null as any, done: false, event }); this.reapOneShots(); return true; } /** Stop one-shots of an asset (or all one-shots when omitted). */ stopSound(idOrName?: string, fade = 0.05) { if (recorder) { const now = recorder.now(); const asset = idOrName ? this.getAsset(idOrName) : undefined; for (const s of this.oneShots) if (!s.done && (!idOrName || s.assetId === asset?.id)) this.stopOneShot(s, now, fade); return; } const ctx = getAudioContext(); if (!ctx) return; const asset = idOrName ? this.getAsset(idOrName) : undefined; for (const s of this.oneShots) { if (!s.done && (!idOrName || s.assetId === asset?.id)) this.stopOneShot(s, ctx.currentTime, fade); } this.reapOneShots(); } stopAll() { for (const key of [...this.timelines.keys()]) this.stopTimeline(key); this.stopSound(); this.stopWatchdog(); } dispose() { this.stopAll(); if (this.master) { try { this.master.disconnect(); } catch { /* gone */ } } this.master = null; if (this.visibilityHandler && typeof document !== 'undefined') document.removeEventListener('visibilitychange', this.visibilityHandler); this.visibilityHandler = null; this.disposed = true; } // -- internals ------------------------------------------------------------------------------ private isStream(asset: ExodeAudioAsset): boolean { if (asset.mode) return asset.mode === 'stream'; return asset.duration > (this.config?.streamThreshold ?? 30); } private ensureMaster() { const ctx = getAudioContext(); if (!ctx || this.master) return; this.master = ctx.createGain(); this.master.connect(ctx.destination); this.applyMaster(); } private applyMaster() { const ctx = getAudioContext(); if (!ctx || !this.master) return; const v = (this.config?.muted || this.host.muted) ? 0 : Math.max(0, this.config?.volume ?? 1) * this.host.volume; this.master.gain.setTargetAtTime(v, ctx.currentTime, 0.015); } private makePanner(ctx: AudioContext, pan: number): StereoPannerNode | null { if (typeof (ctx as any).createStereoPanner !== 'function') return null; const p = ctx.createStereoPanner(); p.pan.value = Math.max(-1, Math.min(1, pan)); return p; } private startClipVoice(ctx: AudioContext, clip: ExodeAudioClip, asset: ExodeAudioAsset, time: number, len: number, rate: number, sig: string): Voice | null { if (recorder) return this.recordClipVoice(clip, asset, time, len, rate, sig); const now = ctx.currentTime; // Clip-relative time at `now`; negative means it starts within the lookahead. const t = time - clip.start; const baseRate = clip.rate && clip.rate > 0 ? clip.rate : 1; const engineRate = rate / baseRate; const startAt = t < 0 ? now + (-t) / engineRate : now; const tFrom = Math.max(0, t); const remaining = len - tFrom; if (remaining <= 0.002) return null; const fade = ctx.createGain(); const vol = ctx.createGain(); // Centered clips skip the panner node: fewer nodes per voice in the audio graph. const pan = clip.pan ? this.makePanner(ctx, clip.pan) : null; fade.connect(vol); if (pan) vol.connect(pan).connect(this.master!); else vol.connect(this.master!); vol.gain.value = clip.muted ? 0 : (clip.volume ?? 1); // Fade / envelope automation, in audio-clock time. const toCtx = (clipT: number) => startAt + (clipT - tFrom) / engineRate; const g0 = clipGainAt(clip, tFrom, len); fade.gain.setValueAtTime(0, startAt); fade.gain.linearRampToValueAtTime(g0, startAt + DECLICK); const points = new Set(); if (clip.fadeIn && clip.fadeIn > tFrom) points.add(clip.fadeIn); if (clip.fadeOut) points.add(Math.max(tFrom, len - clip.fadeOut)); points.add(len); (clip.volumeEnvelope || []).forEach(p => { if (p.time > tFrom && p.time < len) points.add(p.time); }); [...points].filter(p => p > tFrom + DECLICK).sort((a, b) => a - b).forEach(p => { fade.gain.linearRampToValueAtTime(clipGainAt(clip, Math.min(p, len - 1e-4), len), toCtx(p)); }); const voice: Voice = { kind: 'buffer', clipId: clip.id, sig, ctxAtClipZero: startAt - tFrom / engineRate, engineRate, fade, vol, pan, lastVol: clip.volume ?? 1, lastPan: clip.pan ?? 0, lastMuted: !!clip.muted, ended: false, }; const srcPos = sourcePosition(clip, asset, tFrom); const stopAt = startAt + remaining / engineRate; if (this.isStream(asset)) { const item = takeStream(ctx, asset.src); item.node.connect(fade); item.el.loop = !!clip.loop; item.el.playbackRate = rate; const begin = () => { try { item.el.currentTime = srcPos + Math.max(0, ctx.currentTime - startAt) * rate; } catch { /* metadata pending */ } item.el.play().catch(() => undefined); }; const delayMs = Math.max(0, (startAt - now) * 1000); if (delayMs > 4) setTimeout(() => { if (!voice.ended) begin(); }, delayMs); else begin(); voice.kind = 'stream'; voice.el = item.el; voice.elNode = item.node; (voice as any).stream = item; (voice as any).src = asset.src; return voice; } const buffer = cachedBuffer(asset.src); if (!buffer) { loadAudioBuffer(asset.src); fade.disconnect(); vol.disconnect(); pan?.disconnect(); return null; } const src = ctx.createBufferSource(); src.buffer = buffer; src.playbackRate.value = rate; if (clip.loop) { src.loop = true; src.loopStart = Math.min(buffer.duration, (asset.startPad || 0) + (clip.offset || 0)); src.loopEnd = loopEndOf(asset, buffer); } src.connect(fade); src.onended = () => { voice.ended = true; }; src.start(startAt, Math.max(0, Math.min(buffer.duration - 0.001, srcPos))); src.stop(stopAt + DECLICK); voice.source = src; return voice; } private recordClipVoice(clip: ExodeAudioClip, asset: ExodeAudioAsset, time: number, len: number, rate: number, sig: string): Voice | null { const now = recorder!.now(); const baseRate = clip.rate && clip.rate > 0 ? clip.rate : 1; const engineRate = rate / baseRate; const t = time - clip.start; const tFrom = Math.max(0, t); if (len - tFrom <= 0.002) return null; const at = t < 0 ? now + (-t) / engineRate : now; const event: AudioRenderEvent = { src: asset.src, at, offset: sourcePosition(clip, asset, tFrom), dur: (len - tFrom) / engineRate, rate, volume: clip.volume ?? 1, pan: clip.pan ?? 0, loop: clip.loop ? { start: (asset.startPad || 0) + (clip.offset || 0), end: (asset.startPad || 0) + asset.duration } : undefined, gain: gainPoints(clip, len, tFrom, len, at, engineRate), }; recorder!.events.push(event); return { kind: 'record', event, clipId: clip.id, sig, ctxAtClipZero: at - tFrom / engineRate, engineRate, fade: null as any, vol: null as any, pan: null, lastVol: clip.volume ?? 1, lastPan: clip.pan ?? 0, lastMuted: false, ended: false, }; } private applyVoiceMix(v: Voice, volume: number, pan: number, muted: boolean, now: number) { if (v.kind === 'record') return; if (volume !== v.lastVol || muted !== v.lastMuted) { v.vol.gain.setTargetAtTime(muted ? 0 : Math.max(0, volume), now, 0.012); v.lastVol = volume; v.lastMuted = muted; } if (pan !== v.lastPan && v.pan) { v.pan.pan.setTargetAtTime(Math.max(-1, Math.min(1, pan)), now, 0.012); v.lastPan = pan; } } private stopVoice(v: Voice, now: number) { v.ended = true; if (v.kind === 'record') { if (v.event) v.event.dur = Math.max(0, Math.min(v.event.dur, now - v.event.at)); return; } try { v.vol.gain.cancelScheduledValues(now); v.vol.gain.setValueAtTime(v.vol.gain.value, now); v.vol.gain.linearRampToValueAtTime(0, now + DECLICK); } catch { /* context closed */ } if (v.source) { try { v.source.stop(now + DECLICK + 0.002); } catch { /* not started / already stopped */ } const nodes = [v.fade, v.vol, v.pan]; v.source.onended = () => nodes.forEach(n => { try { n?.disconnect(); } catch { /* gone */ } }); } if (v.kind === 'stream' && (v as any).stream) { const item = (v as any).stream; const src = (v as any).src; setTimeout(() => { returnStream(src, item); [v.fade, v.vol, v.pan].forEach(n => { try { n?.disconnect(); } catch { /* gone */ } }); }, DECLICK * 1000 + 10); } } private silenceVoices(tl: TimelineState) { const ctx = recorder ? null : getAudioContext(); const now = recorder ? recorder.now() : ctx ? ctx.currentTime : 0; tl.voices.forEach(v => this.stopVoice(v, now)); tl.voices.clear(); tl.anchor = null; } private stopOneShot(s: OneShot, now: number, fade = DECLICK) { s.done = true; if (s.event) { s.event.dur = Math.max(0, Math.min(s.event.dur, now + fade - s.event.at)); return; } try { s.gain.gain.cancelScheduledValues(now); s.gain.gain.setValueAtTime(s.gain.gain.value, now); s.gain.gain.linearRampToValueAtTime(0, now + fade); } catch { /* closed */ } if (s.source) { try { s.source.stop(now + fade + 0.002); } catch { /* stopped */ } } if (s.el) { const item = (s as any).stream; const asset = this.assets.get(s.assetId); setTimeout(() => { if (item && asset) returnStream(asset.src, item); }, fade * 1000 + 10); } } private reapOneShots() { if (this.oneShots.length > 16) this.oneShots = this.oneShots.filter(s => !s.done); } /** Silence timelines whose engine stopped calling sync (paused RAF, unmounted canvas). */ private startWatchdog() { if (this.watchdog || typeof setInterval === 'undefined') return; this.watchdog = setInterval(() => { const wall = performance.now(); for (const [key, tl] of this.timelines) { if (wall - tl.lastWall > WATCHDOG_MS) this.stopTimeline(key); } if (!this.timelines.size) this.stopWatchdog(); }, WATCHDOG_MS); } private stopWatchdog() { if (this.watchdog) clearInterval(this.watchdog); this.watchdog = null; } private installVisibility() { if (this.visibilityHandler || typeof document === 'undefined') return; this.visibilityHandler = () => { if (document.hidden) this.stopAll(); }; document.addEventListener('visibilitychange', this.visibilityHandler); } }