/** * modelark-references.ts — turn a scene's reference sources into the URLs * ModelArk can actually read, and refuse anything the vendor would reject * BEFORE a byte is uploaded or a cent is spent. * * ModelArk fetches inputs by URL; it cannot read a local path. Three forms * reach it: a public `http(s)://` URL, a `data:;base64,` string (the * vendor says "not for large files"; the whole request body is capped at * 64 MB), and its own `asset://` digital-character URI. So: * - `http(s)://` and `asset://` pass through untouched; * - a local image up to {@link MODELARK_INLINE_IMAGE_MAX_BYTES} is inlined, * as long as the scene's inlined bytes stay under * {@link MODELARK_INLINE_BUDGET_BYTES}; larger images are hosted; * - local video and audio are always hosted (`hostPublicUrl`, the same * temporary host `finish` and `lipsync` already use — ~3 h TTL, 128 MiB). * Size and duration caps are checked against the vendor's limits first, over * the whole scene, so a bad reference never leaves an orphaned upload behind. * Every dependency is injectable so the decision is unit-testable offline. */ import { readFile, stat } from 'node:fs/promises'; import { extname } from 'node:path'; import { hostPublicUrl } from './public-host.js'; import { probeMedia } from '../final-media.js'; import type { ModelArkModelLimits, ModelArkReferencePlan } from './modelark.js'; /** A local image at or below this size is inlined as base64 instead of hosted. */ export const MODELARK_INLINE_IMAGE_MAX_BYTES = 4 * 1024 * 1024; /** Raw bytes one scene may inline in total (well under the vendor's 64 MB request cap after base64 growth). */ export const MODELARK_INLINE_BUDGET_BYTES = 32 * 1024 * 1024; /** Vendor input caps (create-task reference, 2026-09-21). */ export const MODELARK_INPUT_LIMITS = { imageMaxBytes: 30 * 1024 * 1024, videoMaxBytes: 200 * 1024 * 1024, audioMaxBytes: 15 * 1024 * 1024, } as const; export interface ModelArkReferenceDeps { /** Upload a local file to a public URL. Default: `hostPublicUrl`. */ hostFile: (path: string) => Promise; statSize: (path: string) => Promise; readBytes: (path: string) => Promise; /** * Length of a video/audio source in seconds — a local file or a remote * `http(s)://` URL (ffprobe by default, which reads both). May throw; the * caller words the refusal. A remote probe is bounded by * `MODELARK_REMOTE_PROBE_TIMEOUT_MS`. */ probeDurationSeconds: (path: string) => Promise; /** * A video source's stored width, height and frame rate (ffprobe by default, * sharing one probe with the length). May throw; the caller words the * refusal. Only called for video references. */ probeVideoShape: (path: string) => Promise; } export interface ModelArkVideoShape { width?: number; height?: number; /** Nominal frame rate (`r_frame_rate`). */ fps?: number; /** Average frame rate (`avg_frame_rate`); differs from `fps` on variable-frame-rate footage. */ avgFps?: number; /** Normalised container: mp4, mov, mkv, webm, avi, … */ container?: string; videoCodec?: string; audioCodec?: string; } /** The slowest rate measured accepted (24000/1001); the floor sits just under it. */ export const MODELARK_MEASURED_MIN_FPS = 23.976; /** * The reference-video limits from the vendor's two pages (the API reference's * "Video requirements" with its container/encoding table, and the Seedance 2.5 * tutorial, read 2026-09-22). Both state the dimensions, pixel count, aspect, * frame rate, and .mp4/.mov with H.264/H.265 video and AAC/MP3 audio. PCM audio * in .mov comes from the tutorial alone (it is also what ModelArk's own .mov * output carries); the API reference lists AAC/MP3 only, so PCM is accepted in * .mov on the tutorial's word. The pages also disagree on the resolution TIER * (API reference: 480p, 720p, 1080p, 4k; tutorial: 480p, 720p), so the tier is * left to the vendor. */ export const MODELARK_VIDEO_SHAPE_LIMITS = { minSide: 300, maxSide: 6000, minPixels: 407_696, maxPixels: 8_295_044, minAspect: 0.4, maxAspect: 2.5, /** * The vendor's pages say 24–60. 23.976 (24000/1001, the standard film * cadence, and what every 24p camera and NLE actually writes) was MEASURED * accepted on 2026-09-23: task `cgt-20260923053323-d63gs`, Seedance 2.5 in * omni mode with a 720p reference, rendered — see * `docs/audits/2026-09-23-live-acceptance.md`. This table gates the 2.0 * models too, which share the floor untested; the asymmetry is what makes * that acceptable, since this check runs BEFORE submit and a shape the * vendor refuses comes back as a free 400, while the old floor bounced valid * 24p footage with a lossy re-encode instruction. Nothing below 23.9 opens up. */ minFps: 23.9, maxFps: 60, containers: ['mp4', 'mov'], videoCodecs: ['h264', 'hevc'], /** Per container: .mp4 takes AAC or MP3; .mov also takes PCM (the tutorial's table; the API reference lists AAC/MP3 only). */ audioCodecs: { mp4: ['aac', 'mp3'], mov: ['aac', 'mp3', 'pcm'] }, } as const; /** ffprobe's `r_frame_rate` ("24/1", "30000/1001") as a number; undefined when unreadable. */ export function parseFrameRate(value: string | undefined): number | undefined { if (!value) return undefined; const [num, den] = value.split('/').map(Number); const fps = den === undefined ? num : num / den; return Number.isFinite(fps) && fps > 0 ? fps : undefined; } /** Every way a reference video's shape breaks the vendor's limits, worded for the operator; empty when it fits. */ export function modelArkVideoShapeProblems(shape: ModelArkVideoShape): string[] { const L = MODELARK_VIDEO_SHAPE_LIMITS; const { width, height, fps } = shape; if (!width || !height) return ['its dimensions could not be read']; const problems: string[] = []; if (Math.min(width, height) < L.minSide || Math.max(width, height) > L.maxSide) problems.push(`${width}×${height} has a side outside ${L.minSide}–${L.maxSide} px`); const pixels = width * height; if (pixels < L.minPixels || pixels > L.maxPixels) { const hint = pixels < L.minPixels ? "the floor is 614×664; ModelArk's own 480p sizes such as 854×480 or 640×640 fit, a 640×480 or 720×480 SD clip does not" : 'the ceiling is 3326×2494; UHD 3840×2160 fits, DCI 4K 4096×2160 does not'; problems.push(`${width}×${height} is ${pixels.toLocaleString('en-US')} pixels, outside ${L.minPixels.toLocaleString('en-US')}–${L.maxPixels.toLocaleString('en-US')} (${hint})`); } const aspect = width / height; if (aspect < L.minAspect || aspect > L.maxAspect) problems.push(`its aspect ratio ${aspect.toFixed(2)} is outside ${L.minAspect}–${L.maxAspect}`); const { container, videoCodec, audioCodec, avgFps } = shape; if (!container) problems.push('its container could not be read'); // ffprobe names every Matroska-family file `matroska,webm`, so .mkv and .webm cannot be told apart here. else if (!(L.containers as readonly string[]).includes(container)) problems.push(`its container is ${container === 'mkv' || container === 'webm' ? 'Matroska/WebM' : container}; ModelArk takes .mp4 or .mov`); if (!videoCodec) problems.push('its video codec could not be read'); else if (!(L.videoCodecs as readonly string[]).includes(videoCodec)) problems.push(`its video codec is ${videoCodec}; ModelArk takes H.264 or H.265`); if (audioCodec && (container === 'mp4' || container === 'mov')) { const allowed: readonly string[] = L.audioCodecs[container]; const family = audioCodec.startsWith('pcm_') ? 'pcm' : audioCodec; if (!allowed.includes(family)) problems.push(`its audio codec is ${audioCodec}; ModelArk takes AAC or MP3 in .mp4 (and PCM in .mov)`); } // Timebase rounding moves a constant-rate clip's average by well under 1% // (a 1/1000 track, a Matroska round-trip, a copy-concat); that is not VFR. if (fps !== undefined && fps >= L.minFps && fps <= L.maxFps && avgFps !== undefined && Math.abs(avgFps - fps) / fps > 0.02 && (avgFps < L.minFps || avgFps > L.maxFps)) { problems.push(`it plays at ${Number(fps.toFixed(3))} fps nominally but averages ${Number(avgFps.toFixed(3))} fps (variable frame rate), outside ${L.minFps}–${L.maxFps}; re-encode at a constant rate: ffmpeg -i in.mp4 -vf fps=${Math.round(fps)} -c:a copy out.mp4`); } if (fps === undefined) problems.push('its frame rate could not be read'); else if (fps < L.minFps || fps > L.maxFps) problems.push(`${Number(fps.toFixed(3))} fps is outside ${L.minFps}–${L.maxFps}${fps < L.minFps ? ` (${MODELARK_MEASURED_MIN_FPS} is accepted) — re-encode at 24 fps: ffmpeg -i in.mp4 -vf fps=24 -c:a copy out.mp4` : ''}`); return problems; } /** How long a remote reference may take to answer ffprobe before the submit refuses it. */ export const MODELARK_REMOTE_PROBE_TIMEOUT_MS = 20_000; /** * Words a failed probe. Node's execFile timeout rejects with `killed: true` * and a message that never says "timeout" (and an empty stderr on a hang), so * without this a dropped host reads exactly like a refused connection. */ export function describeProbeFailure(error: unknown, timeoutMs: number): string { const killed = typeof error === 'object' && error !== null && (error as { killed?: unknown }).killed === true; if (killed) return `probe timed out after ${Number.isInteger(timeoutMs / 1000) ? timeoutMs / 1000 : (timeoutMs / 1000).toFixed(1)} s`; const message = error instanceof Error ? error.message : String(error); return message.split('\n').map((line) => line.trim()).filter(Boolean).slice(-1)[0] ?? message; } export function defaultModelArkReferenceDeps( env: NodeJS.ProcessEnv = process.env, options: { remoteProbeTimeoutMs?: number } = {}, ): ModelArkReferenceDeps { const remoteProbeTimeoutMs = options.remoteProbeTimeoutMs ?? MODELARK_REMOTE_PROBE_TIMEOUT_MS; // One ffprobe per source serves both the length and the shape (a remote // source is read over the network, so it is not read twice). const probes = new Map>>>(); const probe = async (path: string) => { const remote = /^https?:\/\//i.test(path); if (!probes.has(path)) probes.set(path, probeMedia(path, remote ? { timeoutMs: remoteProbeTimeoutMs, ffprobeBin: env.VCLAW_FFPROBE_BIN } : { ffprobeBin: env.VCLAW_FFPROBE_BIN })); try { return await probes.get(path)!; } catch (error) { throw new Error(describeProbeFailure(error, remoteProbeTimeoutMs)); } }; return { hostFile: async (path) => (await hostPublicUrl(path, { env })).url, statSize: async (path) => (await stat(path)).size, readBytes: (path) => readFile(path), probeDurationSeconds: async (path) => (await probe(path)).durationSeconds, probeVideoShape: async (path) => { const media = await probe(path); return { width: media.width, height: media.height, fps: parseFrameRate(media.frameRate), avgFps: parseFrameRate(media.avgFrameRate), container: media.container, videoCodec: media.videoCodec, audioCodec: media.audioCodec, }; }, }; } /** Probe through the dep and turn any failure into the route's own refusal, scene-attributed. */ async function measuredSeconds(deps: ModelArkReferenceDeps, sceneIndex: number, ref: ModelArkReferencePlan, remote: boolean): Promise { let seconds: number | undefined; try { seconds = await deps.probeDurationSeconds(ref.source); } catch (error) { throw new Error(`seedance-modelark scene ${sceneIndex}: could not measure the length of ${remote ? 'remote ' : ''}${ref.kind} reference ${ref.source} (${error instanceof Error ? error.message : String(error)}); ModelArk caps reference length, so an unmeasured clip is refused${remote ? ' — keep it local or reachable' : ''}.`); } if (seconds === undefined || !Number.isFinite(seconds)) { throw new Error(`seedance-modelark scene ${sceneIndex}: could not measure the length of ${remote ? 'remote ' : ''}${ref.kind} reference ${ref.source}; ModelArk caps reference length, so an unmeasured clip is refused${remote ? ' — keep it local or reachable' : ''}.`); } return seconds; } /** Refuse, before any upload, a reference video whose shape the vendor documents as unacceptable. */ async function assertVideoShape(deps: ModelArkReferenceDeps, sceneIndex: number, ref: ModelArkReferencePlan, remote: boolean): Promise { if (ref.kind !== 'video') return; let shape: ModelArkVideoShape | undefined; try { shape = await deps.probeVideoShape(ref.source); } catch (error) { throw new Error(`seedance-modelark scene ${sceneIndex}: could not read the shape of ${remote ? 'remote ' : ''}video reference ${ref.source} (${error instanceof Error ? error.message : String(error)}); ModelArk limits reference-video dimensions and frame rate, so an unread clip is refused.`); } const problems = modelArkVideoShapeProblems(shape ?? {}); if (problems.length > 0) { throw new Error(`seedance-modelark scene ${sceneIndex}: reference video ${ref.source} cannot be sent to ModelArk: ${problems.join('; ')}.`); } } export interface MaterializedModelArkReference { source: string; url: string; delivery: 'passthrough' | 'inline' | 'hosted'; } const IMAGE_MIME: Record = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp', '.gif': 'image/gif', '.bmp': 'image/bmp', '.heic': 'image/heic', '.heif': 'image/heif', }; export function isRemoteModelArkReference(source: string): boolean { return /^https?:\/\//i.test(source) || source.startsWith('asset://'); } function maxBytesFor(kind: ModelArkReferencePlan['kind']): number { if (kind === 'image') return MODELARK_INPUT_LIMITS.imageMaxBytes; if (kind === 'video') return MODELARK_INPUT_LIMITS.videoMaxBytes; return MODELARK_INPUT_LIMITS.audioMaxBytes; } /** * Resolve every planned reference of one scene, in order. Throws before any * upload when a local file is missing, over the vendor's size cap, a video is * shorter than the model allows, or the combined video+audio length exceeds * the model's reference budget. A remote `http(s)://` video or audio source * is MEASURED too (ffprobe reads a URL), since the vendor's reference-seconds * budget counts it the same as a local clip: before #604 item 8 it contributed * 0 here and the vendor refused at create instead of this preflight. Its byte * size stays the vendor's to check. Every video reference (local or remote) is * also held to the vendor's dimension, pixel-count, aspect and frame-rate * limits (`MODELARK_VIDEO_SHAPE_LIMITS`). A ModelArk `asset://` source cannot * be probed and is trusted as given. */ export async function materializeModelArkReferences( sceneIndex: number, refs: readonly ModelArkReferencePlan[], limits: Pick, deps: ModelArkReferenceDeps, ): Promise { const sizes = new Map(); let referenceSeconds = 0; // Pass 1: every check; nothing is written, inlined or uploaded (a remote // http(s) video/audio clip IS read over the network to measure it, bounded // by the probe timeout). Remote clips are never sized; asset:// is neither. for (const ref of refs) { if (isRemoteModelArkReference(ref.source)) { if (!/^https?:\/\//i.test(ref.source) || (ref.kind !== 'video' && ref.kind !== 'audio')) continue; const seconds = await measuredSeconds(deps, sceneIndex, ref, true); await assertVideoShape(deps, sceneIndex, ref, true); if (ref.kind === 'video' && seconds < limits.minReferenceVideoSeconds) { throw new Error(`seedance-modelark scene ${sceneIndex}: reference video ${ref.source} is ${seconds.toFixed(1)} s; the model needs at least ${limits.minReferenceVideoSeconds} s.`); } referenceSeconds += seconds; continue; } let size: number; try { size = await deps.statSize(ref.source); } catch (error) { throw new Error(`seedance-modelark scene ${sceneIndex}: reference ${ref.source} cannot be read (${error instanceof Error ? error.message : String(error)}).`); } sizes.set(ref.source, size); const cap = maxBytesFor(ref.kind); if (size > cap) { throw new Error(`seedance-modelark scene ${sceneIndex}: ${ref.kind} reference ${ref.source} is ${size} bytes; ModelArk accepts at most ${cap}.`); } if (ref.kind === 'video' || ref.kind === 'audio') { const seconds = await measuredSeconds(deps, sceneIndex, ref, false); await assertVideoShape(deps, sceneIndex, ref, false); if (ref.kind === 'video' && seconds < limits.minReferenceVideoSeconds) { throw new Error(`seedance-modelark scene ${sceneIndex}: reference video ${ref.source} is ${seconds.toFixed(1)} s; the model needs at least ${limits.minReferenceVideoSeconds} s.`); } referenceSeconds += seconds; } } if (referenceSeconds > limits.maxReferenceSeconds) { throw new Error( `seedance-modelark scene ${sceneIndex}: reference video+audio total ${referenceSeconds.toFixed(1)} s exceeds the model's ${limits.maxReferenceSeconds} s budget by ${(referenceSeconds - limits.maxReferenceSeconds).toFixed(2)} s (in VCLAW_MODELARK_CHAIN_MODE=extend the previous clip counts toward it).`, ); } // Pass 2: inline or host. const out: MaterializedModelArkReference[] = []; let inlined = 0; for (const ref of refs) { if (isRemoteModelArkReference(ref.source)) { out.push({ source: ref.source, url: ref.source, delivery: 'passthrough' }); continue; } const size = sizes.get(ref.source) ?? 0; const mime = IMAGE_MIME[extname(ref.source).toLowerCase()]; const canInline = ref.kind === 'image' && mime !== undefined && size <= MODELARK_INLINE_IMAGE_MAX_BYTES && inlined + size <= MODELARK_INLINE_BUDGET_BYTES; if (canInline) { const bytes = await deps.readBytes(ref.source); inlined += bytes.length; out.push({ source: ref.source, url: `data:${mime};base64,${bytes.toString('base64')}`, delivery: 'inline' }); continue; } const url = await deps.hostFile(ref.source); out.push({ source: ref.source, url, delivery: 'hosted' }); } return out; }