import { createHash } from "node:crypto"; import { isDeepStrictEqual } from "node:util"; import { readFile } from "node:fs/promises"; import type { ArollSegment, BgmPlanReceipt, BgmTrack, FileRef, TalkingHeadProject, TalkingHeadSnapshot, } from "./contracts.ts"; import { timelineDuration } from "./transcript.ts"; import { resolveExistingWorkspaceFile, snapshotFile } from "./workspace.ts"; interface AudioAnalysisManifest { schemaVersion: 1; standard: "EBU-R128/ITU-R-BS.1770"; source: FileRef; ranges: Array<{ startSeconds: number; endSeconds: number }>; analyzedDurationSeconds: number; loudness: { integratedLufs: number; loudnessRangeLu: number; truePeakDbtp: number; }; artifacts: Array; } export interface PlanBgmInput { aroll: ArollSegment[]; voiceAnalysisPath: string; musicAnalysisPath: string; targetMusicBelowDialogueLu: number; } export interface SelectBgmInput { aroll: ArollSegment[]; planReceipt: BgmPlanReceipt; selectedMusicAnalysisPath: string; selectedStartMs: number; selectedEndMs: number; evidenceTimestampsMs: number[]; fadeInMs: number; fadeOutMs: number; } function sha256(value: unknown): string { return createHash("sha256").update(JSON.stringify(value)).digest("hex"); } function assertFileRef(value: unknown, label: string): asserts value is FileRef { if (!value || typeof value !== "object") throw new Error(`Invalid ${label} file reference`); const ref = value as Partial; if (typeof ref.path !== "string" || !Number.isInteger(ref.bytes) || (ref.bytes ?? -1) < 0 || typeof ref.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(ref.sha256)) { throw new Error(`Invalid ${label} file reference`); } } async function readAnalysis(cwd: string, path: string, label: string): Promise<{ manifest: AudioAnalysisManifest; ref: FileRef }> { const absolute = await resolveExistingWorkspaceFile(cwd, path); let value: unknown; try { value = JSON.parse(await readFile(absolute, "utf8")); } catch (error) { throw new Error(`Invalid ${label} audio analysis: ${(error as Error).message}`); } const manifest = value as Partial; if (manifest.schemaVersion !== 1 || manifest.standard !== "EBU-R128/ITU-R-BS.1770" || !Array.isArray(manifest.ranges) || manifest.ranges.length < 1 || !Array.isArray(manifest.artifacts) || manifest.artifacts.length < 1 || !manifest.loudness || !Number.isFinite(manifest.loudness.integratedLufs) || !Number.isFinite(manifest.loudness.truePeakDbtp)) { throw new Error(`Invalid ${label} audio analysis manifest`); } assertFileRef(manifest.source, `${label} source`); for (const artifact of manifest.artifacts) { assertFileRef(artifact, `${label} waveform artifact`); if (!Number.isFinite(artifact.startSeconds) || !Number.isFinite(artifact.endSeconds) || artifact.endSeconds <= artifact.startSeconds) { throw new Error(`Invalid ${label} waveform artifact range`); } } return { manifest: manifest as AudioAnalysisManifest, ref: await snapshotFile(cwd, path) }; } function sameFile(left: FileRef, right: FileRef): boolean { return left.path === right.path && left.bytes === right.bytes && left.sha256 === right.sha256; } function expectedVoiceRanges(aroll: ArollSegment[]): AudioAnalysisManifest["ranges"] { return aroll.map((segment) => ({ startSeconds: segment.sourceStartMs / 1_000, endSeconds: segment.sourceEndMs / 1_000 })); } function assertRangesEqual(actual: AudioAnalysisManifest["ranges"], expected: AudioAnalysisManifest["ranges"]): void { if (actual.length !== expected.length) throw new Error("Voice analysis ranges must exactly match the planned A-roll"); for (let index = 0; index < expected.length; index += 1) { if (Math.abs(actual[index]!.startSeconds - expected[index]!.startSeconds) > 0.001 || Math.abs(actual[index]!.endSeconds - expected[index]!.endSeconds) > 0.001) { throw new Error("Voice analysis ranges must exactly match the planned A-roll"); } } } function receiptPayload(receipt: BgmPlanReceipt): Omit { const { planSha256: _planSha256, ...payload } = receipt; return payload; } function assertPlanReceipt(receipt: BgmPlanReceipt, projectId: string, revision: number, aroll: ArollSegment[]): void { if (receipt.schemaVersion !== 1 || receipt.projectId !== projectId || receipt.revision !== revision) { throw new Error("BGM plan receipt does not belong to this project revision"); } if (receipt.arollSha256 !== sha256(aroll)) throw new Error("BGM plan receipt does not match the planned A-roll"); if (receipt.planSha256 !== sha256(receiptPayload(receipt))) throw new Error("BGM plan receipt hash is invalid"); if (receipt.outputDurationMs !== timelineDuration(aroll)) throw new Error("BGM plan output duration does not match its A-roll"); const delta = receipt.musicDurationMs - receipt.outputDurationMs; const relation = Math.abs(delta) <= 1 ? "equal" : delta > 0 ? "longer" : "shorter"; if (receipt.durationRelation !== relation || receipt.requiredPlayback !== (relation === "shorter" ? "loop" : "trim")) { throw new Error("BGM plan duration relation or playback strategy is invalid"); } if (receipt.targetMusicBelowDialogueLu < 6 || receipt.targetMusicBelowDialogueLu > 30 || receipt.maxTruePeakDbtp < -6 || receipt.maxTruePeakDbtp > -0.1) { throw new Error("BGM plan mix policy is outside the supported range"); } } async function assertPlanFilesCurrent( cwd: string, project: TalkingHeadProject, aroll: ArollSegment[], receipt: BgmPlanReceipt, ): Promise { const voiceAnalysis = await snapshotFile(cwd, receipt.voiceAnalysis.path); if (!sameFile(voiceAnalysis, receipt.voiceAnalysis)) throw new Error("Voice analysis manifest changed after BGM planning"); const musicAnalysis = await snapshotFile(cwd, receipt.musicAnalysis.path); if (!sameFile(musicAnalysis, receipt.musicAnalysis)) throw new Error("BGM analysis manifest changed after BGM planning"); const musicAsset = await snapshotFile(cwd, receipt.musicAsset.path); if (!sameFile(musicAsset, receipt.musicAsset)) throw new Error("BGM asset changed after BGM planning"); const voice = (await readAnalysis(cwd, receipt.voiceAnalysis.path, "voice")).manifest; const music = (await readAnalysis(cwd, receipt.musicAnalysis.path, "BGM")).manifest; if (!sameFile(voice.source, project.source)) throw new Error("Voice analysis source does not match the talking-head source"); assertRangesEqual(voice.ranges, expectedVoiceRanges(aroll)); if (voice.loudness.integratedLufs !== receipt.dialogueIntegratedLufs) { throw new Error("BGM plan dialogue loudness does not match its analysis manifest"); } if (!sameFile(music.source, receipt.musicAsset) || music.ranges.length !== 1 || Math.abs(music.ranges[0]!.startSeconds) > 0.001 || Math.abs(music.ranges[0]!.endSeconds * 1_000 - receipt.musicDurationMs) > 1 || music.loudness.integratedLufs !== receipt.fullMusicIntegratedLufs) { throw new Error("BGM plan duration or loudness does not match its complete analysis manifest"); } return music; } export async function planBgm( cwd: string, project: TalkingHeadProject, snapshot: TalkingHeadSnapshot, input: PlanBgmInput, ): Promise<{ plan: { outputDurationMs: number; musicDurationMs: number; durationRelation: "longer" | "equal" | "shorter"; requiredPlayback: "trim" | "loop"; preliminaryGainDb: number; }; planReceipt: BgmPlanReceipt; }> { if (snapshot.projectId !== project.projectId || snapshot.revision !== project.currentRevision) { throw new Error("BGM planning requires the current talking-head revision"); } if (input.aroll.length < 1) throw new Error("BGM planning requires at least one A-roll segment"); if (!Number.isFinite(input.targetMusicBelowDialogueLu) || input.targetMusicBelowDialogueLu < 6 || input.targetMusicBelowDialogueLu > 30) { throw new Error("BGM target must be 6-30 LU below dialogue"); } const voice = await readAnalysis(cwd, input.voiceAnalysisPath, "voice"); const music = await readAnalysis(cwd, input.musicAnalysisPath, "BGM"); if (voice.manifest.source.path !== project.source.path || voice.manifest.source.sha256 !== project.source.sha256 || voice.manifest.source.bytes !== project.source.bytes) { throw new Error("Voice analysis source does not match the talking-head source"); } assertRangesEqual(voice.manifest.ranges, expectedVoiceRanges(input.aroll)); if (music.manifest.ranges.length !== 1 || Math.abs(music.manifest.ranges[0]!.startSeconds) > 0.001) { throw new Error("BGM analysis must cover the complete music asset from timestamp zero"); } const currentMusic = await snapshotFile(cwd, music.manifest.source.path); if (!sameFile(currentMusic, music.manifest.source)) throw new Error("BGM analysis source changed after analysis"); const outputDurationMs = timelineDuration(input.aroll); const musicDurationMs = Math.round(music.manifest.ranges[0]!.endSeconds * 1_000); const delta = musicDurationMs - outputDurationMs; const durationRelation = Math.abs(delta) <= 1 ? "equal" : delta > 0 ? "longer" : "shorter"; const requiredPlayback = durationRelation === "shorter" ? "loop" : "trim"; const dialogueIntegratedLufs = voice.manifest.loudness.integratedLufs; const fullMusicIntegratedLufs = music.manifest.loudness.integratedLufs; if (dialogueIntegratedLufs < -70 || dialogueIntegratedLufs > 0 || fullMusicIntegratedLufs < -70 || fullMusicIntegratedLufs > 0) { throw new Error("BGM planning requires finite Integrated LUFS measurements between -70 and 0"); } const gainDb = dialogueIntegratedLufs - input.targetMusicBelowDialogueLu - fullMusicIntegratedLufs; if (gainDb < -60 || gainDb > 24) throw new Error("Measured BGM gain falls outside the supported -60 to 24 dB range"); const payload: Omit = { schemaVersion: 1, projectId: project.projectId, revision: snapshot.revision, arollSha256: sha256(input.aroll), outputDurationMs, musicDurationMs, durationRelation, requiredPlayback, voiceAnalysis: voice.ref, musicAnalysis: music.ref, musicAsset: currentMusic, dialogueIntegratedLufs, fullMusicIntegratedLufs, targetMusicBelowDialogueLu: input.targetMusicBelowDialogueLu, maxTruePeakDbtp: -1, }; const planReceipt: BgmPlanReceipt = { ...payload, planSha256: sha256(payload) }; return { plan: { outputDurationMs, musicDurationMs, durationRelation, requiredPlayback, preliminaryGainDb: gainDb, }, planReceipt, }; } async function assertVisualEvidence( cwd: string, manifest: AudioAnalysisManifest, selectedStartMs: number, selectedEndMs: number, evidenceTimestampsMs: number[], ): Promise { if (evidenceTimestampsMs.length < 2 || evidenceTimestampsMs.length > 100 || evidenceTimestampsMs.some((timestamp) => !Number.isFinite(timestamp) || timestamp < 0)) { throw new Error("BGM selection requires 2-100 waveform evidence timestamps"); } if (!evidenceTimestampsMs.some((timestamp) => Math.abs(timestamp - selectedStartMs) <= 1) || !evidenceTimestampsMs.some((timestamp) => Math.abs(timestamp - selectedEndMs) <= 1)) { throw new Error("BGM waveform evidence must include the selected start and end timestamps"); } const checked = new Set(); for (const timestampMs of evidenceTimestampsMs) { const timestamp = timestampMs / 1_000; const artifact = manifest.artifacts.find((candidate) => timestamp >= candidate.startSeconds - 0.001 && timestamp <= candidate.endSeconds + 0.001); if (!artifact) throw new Error(`BGM waveform evidence timestamp ${timestampMs}ms is not covered by the analysis images`); if (checked.has(artifact.path)) continue; const current = await snapshotFile(cwd, artifact.path); if (!sameFile(current, artifact)) throw new Error(`BGM waveform artifact changed after analysis: ${artifact.path}`); checked.add(artifact.path); } } function assertSelectionRules( plan: BgmPlanReceipt, selectedStartMs: number, selectedEndMs: number, fadeInMs: number, fadeOutMs: number, ): void { if (!Number.isFinite(selectedStartMs) || selectedStartMs < 0 || !Number.isFinite(selectedEndMs) || selectedEndMs <= selectedStartMs || selectedEndMs > plan.musicDurationMs + 1) { throw new Error("Invalid BGM source selection range"); } const selectedDurationMs = selectedEndMs - selectedStartMs; if (plan.requiredPlayback === "trim" && Math.abs(selectedDurationMs - plan.outputDurationMs) > 1) { throw new Error("Long BGM selection must provide one source window exactly matching the output duration"); } if (plan.requiredPlayback === "loop" && (selectedDurationMs < 500 || selectedDurationMs >= plan.outputDurationMs)) { throw new Error("Short BGM loop selection must be at least 500ms and shorter than the output"); } if (!Number.isFinite(fadeInMs) || fadeInMs < 0 || !Number.isFinite(fadeOutMs) || fadeOutMs < 250 || fadeInMs + fadeOutMs > plan.outputDurationMs) { throw new Error("BGM requires a fade-out of at least 250ms and valid fade durations"); } } function assertSelectedMusicAnalysis( manifest: AudioAnalysisManifest, plan: BgmPlanReceipt, selectedStartMs: number, selectedEndMs: number, ): void { if (!sameFile(manifest.source, plan.musicAsset) || manifest.ranges.length !== 1 || Math.abs(manifest.ranges[0]!.startSeconds * 1_000 - selectedStartMs) > 1 || Math.abs(manifest.ranges[0]!.endSeconds * 1_000 - selectedEndMs) > 1) { throw new Error("Selected BGM analysis must measure the exact chosen source window"); } } export async function selectBgm( cwd: string, project: TalkingHeadProject, snapshot: TalkingHeadSnapshot, input: SelectBgmInput, ): Promise<{ bgm: BgmTrack }> { assertPlanReceipt(input.planReceipt, project.projectId, snapshot.revision, input.aroll); const manifest = await assertPlanFilesCurrent(cwd, project, input.aroll, input.planReceipt); assertSelectionRules(input.planReceipt, input.selectedStartMs, input.selectedEndMs, input.fadeInMs, input.fadeOutMs); const selectedAnalysis = await readAnalysis(cwd, input.selectedMusicAnalysisPath, "selected BGM"); assertSelectedMusicAnalysis(selectedAnalysis.manifest, input.planReceipt, input.selectedStartMs, input.selectedEndMs); const mix = { method: "ebu-r128-dialogue-relative" as const, dialogueAnalysisPath: input.planReceipt.voiceAnalysis.path, musicAnalysisPath: selectedAnalysis.ref.path, dialogueIntegratedLufs: input.planReceipt.dialogueIntegratedLufs, musicIntegratedLufs: selectedAnalysis.manifest.loudness.integratedLufs, targetMusicBelowDialogueLu: input.planReceipt.targetMusicBelowDialogueLu, maxTruePeakDbtp: input.planReceipt.maxTruePeakDbtp, }; const gainDb = mix.dialogueIntegratedLufs - mix.targetMusicBelowDialogueLu - mix.musicIntegratedLufs; if (mix.musicIntegratedLufs < -70 || mix.musicIntegratedLufs > 0 || gainDb < -60 || gainDb > 24) { throw new Error("Selected BGM loudness produces an unsupported measured gain"); } await assertVisualEvidence(cwd, manifest, input.selectedStartMs, input.selectedEndMs, input.evidenceTimestampsMs); const selectionPayload = { planSha256: input.planReceipt.planSha256, selectedMusicAnalysis: selectedAnalysis.ref, mix, selectedStartMs: input.selectedStartMs, selectedEndMs: input.selectedEndMs, evidenceTimestampsMs: input.evidenceTimestampsMs, fadeInMs: input.fadeInMs, fadeOutMs: input.fadeOutMs, }; return { bgm: { assetPath: input.planReceipt.musicAsset.path, assetBytes: input.planReceipt.musicAsset.bytes, assetSha256: input.planReceipt.musicAsset.sha256, sourceStartMs: input.selectedStartMs, sourceEndMs: input.selectedEndMs, outputStartMs: 0, outputEndMs: input.planReceipt.outputDurationMs, playback: input.planReceipt.requiredPlayback, fadeInMs: input.fadeInMs, fadeOutMs: input.fadeOutMs, ducking: "dialogue-sidechain", mix, selectionReceipt: { planReceipt: structuredClone(input.planReceipt), selectedMusicAnalysis: selectedAnalysis.ref, mix, evidenceTimestampsMs: [...input.evidenceTimestampsMs], selectionSha256: sha256(selectionPayload), }, }, }; } export async function verifyBgmSelection( cwd: string, project: TalkingHeadProject, revision: number, aroll: ArollSegment[], bgm: BgmTrack, ): Promise { const receipt = bgm.selectionReceipt; if (!receipt) throw new Error("BGM selection receipt is required"); assertPlanReceipt(receipt.planReceipt, project.projectId, revision, aroll); const manifest = await assertPlanFilesCurrent(cwd, project, aroll, receipt.planReceipt); assertSelectionRules(receipt.planReceipt, bgm.sourceStartMs, bgm.sourceEndMs, bgm.fadeInMs, bgm.fadeOutMs); const selectedAnalysisRef = await snapshotFile(cwd, receipt.selectedMusicAnalysis.path); if (!sameFile(selectedAnalysisRef, receipt.selectedMusicAnalysis)) throw new Error("Selected BGM analysis manifest changed after selection"); const selectedAnalysis = (await readAnalysis(cwd, receipt.selectedMusicAnalysis.path, "selected BGM")).manifest; assertSelectedMusicAnalysis(selectedAnalysis, receipt.planReceipt, bgm.sourceStartMs, bgm.sourceEndMs); if (bgm.assetPath !== receipt.planReceipt.musicAsset.path || bgm.assetBytes !== receipt.planReceipt.musicAsset.bytes || bgm.assetSha256 !== receipt.planReceipt.musicAsset.sha256 || bgm.outputStartMs !== 0 || bgm.outputEndMs !== receipt.planReceipt.outputDurationMs || bgm.playback !== receipt.planReceipt.requiredPlayback || bgm.ducking !== "dialogue-sidechain" || !isDeepStrictEqual(bgm.mix, receipt.mix) || receipt.mix.dialogueIntegratedLufs !== receipt.planReceipt.dialogueIntegratedLufs || receipt.mix.musicIntegratedLufs !== selectedAnalysis.loudness.integratedLufs || receipt.mix.dialogueAnalysisPath !== receipt.planReceipt.voiceAnalysis.path || receipt.mix.musicAnalysisPath !== receipt.selectedMusicAnalysis.path || receipt.mix.targetMusicBelowDialogueLu !== receipt.planReceipt.targetMusicBelowDialogueLu || receipt.mix.maxTruePeakDbtp !== receipt.planReceipt.maxTruePeakDbtp) { throw new Error("BGM selection fields do not match its plan receipt"); } const selectionPayload = { planSha256: receipt.planReceipt.planSha256, selectedMusicAnalysis: receipt.selectedMusicAnalysis, mix: receipt.mix, selectedStartMs: bgm.sourceStartMs, selectedEndMs: bgm.sourceEndMs, evidenceTimestampsMs: receipt.evidenceTimestampsMs, fadeInMs: bgm.fadeInMs, fadeOutMs: bgm.fadeOutMs, }; if (receipt.selectionSha256 !== sha256(selectionPayload)) throw new Error("BGM selection receipt hash is invalid"); await assertVisualEvidence(cwd, manifest, bgm.sourceStartMs, bgm.sourceEndMs, receipt.evidenceTimestampsMs); }