// rnx film — record a flow on the flat canvas, then re-play that // recording on the 3d device stage under a camera cinematic and capture the // stage as video. // // running a flow inside live 3d mode is too slow to look good (the guest // app, the shell, and the WebGL stage all compete for the same cores), so // filming is two-phase: the flow records at full speed on the flat canvas // via the engine recording store, then the 3d recording editor plays the // webm back on the phone screen while only the stage renders. the stage // canvas is captured with MediaRecorder (WebGL content only — the editor's // DOM dock never appears in the capture). import { spawnSync } from 'node:child_process' import { existsSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' import { dirname, extname, join, resolve } from 'node:path' import { callInBridgeWrite, createBridgeFromParsed, parseBridgeCliArgs, type WsBridge, } from '../ws-bridge' import { buildConfigureOptions, waitForThreeModeRuntimeReadyOnBridge } from './three-mode' interface FilmOptions { port?: number verbose?: boolean } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) // flags consumed by film itself; everything else that buildConfigureOptions // understands (--background, --colorway, --env-*, --room, --focus*, ...) // passes through to the stage configure call. const FILM_VALUE_FLAGS = ['--out', '--fps', '--script', '--slow'] const FILM_BOOLEAN_FLAGS = ['--keep-three'] const CONFIGURE_VALUE_FLAGS = [ '--background', '--gradient', '--gradient-angle', '--colorway', '--environment', '--env-shape', '--env-spread', '--env-size', '--env-color', '--env-glow', '--env-plastic', '--room', '--focus', '--focus-amount', '--focus-range', ] const MAX_FILM_MS = 120_000 const STAGE_BITRATE = 16_000_000 function flagValue(args: string[], flag: string): string | undefined { const i = args.indexOf(flag) return i >= 0 ? args[i + 1] : undefined } async function evalInPage(bridge: WsBridge, code: string, timeoutMs = 30_000) { return (await bridge.send({ type: 'evaluate', code }, { timeoutMs })) as T } export async function runFilm(args: string[], opts: FilmOptions): Promise { if (args.length === 0 || args.includes('--help') || args.includes('-h')) { console.log(` rnx film — record a flow flat, replay it on the 3d stage, save a clip usage: rnx film [options] how it works: 1. the flow runs at full speed on the flat canvas while the engine recording store captures the device screen 2. 3d mode + the recording editor come up and play that capture back on the phone screen while a camera script animates 3. the WebGL stage canvas is recorded and saved options: --out output file (.webm, or .mp4 when ffmpeg is installed). default: ~/Desktop/rnx-films/.webm --script camera script to animate during playback (default: hero-arc) --fps stage capture framerate (default: 30) --slow per-step delay forwarded to the flow runner --keep-three leave 3d mode + the recording editor on afterwards --sim target a specific connected sim stage decor (same flags as \`rnx mode three configure\`): --background · --gradient · --gradient-angle --colorway · --room on|off · --focus on|off · --focus-amount --focus-range · --environment on|off · --env-shape/spread/size/color --env-glow on|off · --env-plastic on|off examples: rnx film flows/feed-scroll.yaml rnx film flows/swap.yaml --script hero-arc --background gradient-tide --out ~/Desktop/swap-3d.mp4 rnx film flows/login.yaml --colorway deep-blue --environment on --env-shape pane --room on `) return 0 } const flowPath = args.find((a) => !a.startsWith('--') && /\.ya?ml$/.test(a)) if (!flowPath || !existsSync(resolve(flowPath))) { console.error(` film: flow file not found: ${flowPath ?? '(none given)'}`) return 1 } const fps = Number.parseInt(flagValue(args, '--fps') ?? '30', 10) const scriptId = flagValue(args, '--script') ?? 'hero-arc' const keepThree = args.includes('--keep-three') const flowName = flowPath.replace(/^.*\//, '').replace(/\.ya?ml$/, '') const outArg = flagValue(args, '--out') const outPath = resolve( outArg ?? join(homedir(), 'Desktop', 'rnx-films', `${flowName}.webm`), ) const wantsMp4 = extname(outPath).toLowerCase() === '.mp4' if (wantsMp4 && spawnSync('ffmpeg', ['-version']).status !== 0) { console.error(' film: --out ends in .mp4 but ffmpeg is not installed') console.error(' install ffmpeg or use a .webm output path') return 1 } const parsed = parseBridgeCliArgs(args, { port: opts.port, stripValueFlags: [...FILM_VALUE_FLAGS, ...CONFIGURE_VALUE_FLAGS], stripBooleanFlags: FILM_BOOLEAN_FLAGS, }) const bridge = createBridgeFromParsed({ ...parsed, commandTimeoutMs: 60_000 }) try { // remember the shell state we mutate so a plain run is side-effect free const before = await evalInPage<{ threeMode?: boolean; editor?: boolean }>( bridge, `(() => { const s = window.SootSim?.bridges?.settings?.get?.() || {} return { threeMode: s.threeMode === true, editor: s.threeRecordingEditor === true } })()`, ) // -- phase 1: record the flow on the flat canvas ---------------------- console.log(` film: recording flow ${flowName} on the flat canvas`) const started = await evalInPage( bridge, `(async () => !!(await window.SootSim?.bridges?.startRecording?.('video', undefined, { skipEntitlement: true })))()`, ) if (!started) { console.error( ' film: engine recording store did not start (is @rnx/plugin-recording installed?)', ) return 1 } const t0 = Date.now() await sleep(500) const { runFlowPlayback } = await import('./flow') const playbackArgs = [flowPath] const simFlag = flagValue(args, '--sim') if (simFlag) playbackArgs.push('--sim', simFlag) const slowFlag = flagValue(args, '--slow') if (slowFlag) playbackArgs.push('--slow', slowFlag) const flowCode = await runFlowPlayback(playbackArgs) await sleep(400) const durationMs = Math.min(Date.now() - t0, MAX_FILM_MS) await evalInPage(bridge, `window.SootSim?.bridges?.stopRecording?.()`) if (flowCode !== 0) { console.error(` film: flow failed (exit ${flowCode}) — not filming a broken take`) return flowCode } let hasBlob = false for (let i = 0; i < 50; i++) { await sleep(400) const st = await evalInPage<{ state?: string; hasBlob?: boolean }>( bridge, `(() => { const s = window.SootSim?.bridges?.getRecordingState?.() return { state: s?.state, hasBlob: !!s?.lastVideoBlob } })()`, ) if (st?.state === 'idle' && st.hasBlob) { hasBlob = true break } } if (!hasBlob) { console.error(' film: flow recording never produced a video blob') return 1 } console.log(` film: flow captured (${(durationMs / 1000).toFixed(1)}s)`) // -- phase 2: bring up the 3d stage + recording editor ----------------- await evalInPage( bridge, `window.SootSim?.bridges?.settings?.set?.('threeMode', true)`, ) await waitForThreeModeRuntimeReadyOnBridge(bridge) await sleep(1400) await evalInPage( bridge, `window.SootSim?.bridges?.settings?.set?.('threeRecordingEditor', true)`, ) let replaySeeded = false for (let i = 0; i < 60; i++) { await sleep(250) const seeded = await evalInPage( bridge, `(() => { const rs = window.SootSim?.replayStore return rs?.value?.preview?.surface === 'video' && !!rs?.value?.preview?.flowVideoUrl })()`, ) if (seeded) { replaySeeded = true break } } if (!replaySeeded) { console.error(' film: recording editor did not seed the replay store') return 1 } const configure = buildConfigureOptions(args) delete configure.script configure.resetPose = true await callInBridgeWrite(bridge, 'SootSim.bridges.threeMode.configure', configure) await sleep(2400) // -- phase 3: capture the stage while the replay + camera script run --- console.log(` film: filming the 3d stage (script: ${scriptId})`) const rec = await evalInPage<{ ok: boolean; error?: string }>( bridge, `(() => { const c = window.SootSim?.bridges?.threeMode?.getCanvas?.() if (!c || !c.captureStream) return { ok: false, error: 'no stage canvas' } let mime = 'video/webm;codecs=vp9' if (!('MediaRecorder' in window) || !MediaRecorder.isTypeSupported(mime)) mime = 'video/webm' const stream = c.captureStream(${fps}) const mr = new MediaRecorder(stream, { mimeType: mime, videoBitsPerSecond: ${STAGE_BITRATE} }) const st = { mr, chunks: [], b64: null } mr.ondataavailable = (e) => { if (e.data && e.data.size) st.chunks.push(e.data) } mr.onstop = () => { const blob = new Blob(st.chunks, { type: 'video/webm' }) const fr = new FileReader() fr.onload = () => { st.b64 = String(fr.result).split(',')[1] || '' } fr.readAsDataURL(blob) } window.__sootsimFilm = st mr.start(200) return { ok: true } })()`, ) if (!rec?.ok) { console.error(` film: stage capture failed: ${rec?.error ?? 'unknown'}`) return 1 } await evalInPage( bridge, `((dur) => { const rs = window.SootSim?.replayStore if (!rs) return false const probed = rs.value?.playback?.durationMs const total = probed && probed > 500 ? probed : dur const start = performance.now() const tick = () => { const t = performance.now() - start rs.emit({ ...rs.value, playback: { ...rs.value.playback, playing: true, scrubbing: false, currentTimeMs: Math.min(t, total) } }) if (t < total) requestAnimationFrame(tick) } requestAnimationFrame(tick) return true })(${durationMs})`, ) await callInBridgeWrite(bridge, 'SootSim.bridges.threeMode.configure', { script: { id: scriptId, progress: 1, animate: true, slow: true }, }) await sleep(durationMs + 700) await evalInPage( bridge, `(() => { const s = window.__sootsimFilm; if (s?.mr?.state === 'recording') s.mr.stop(); return true })()`, ) let b64Size = 0 for (let i = 0; i < 60; i++) { await sleep(500) const probe = await evalInPage<{ ready: boolean; size: number }>( bridge, `(() => { const s = window.__sootsimFilm; return { ready: !!(s && s.b64), size: s && s.b64 ? s.b64.length : 0 } })()`, ) if (probe?.ready && probe.size > 0) { b64Size = probe.size break } } if (!b64Size) { console.error(' film: stage recording never finished encoding') return 1 } // stream the base64 back in chunks (bridge messages cap out on very // large single payloads) const CHUNK = 2_400_000 let b64 = '' for (let offset = 0; offset < b64Size; offset += CHUNK) { const part = await evalInPage<{ data: string }>( bridge, `(() => ({ data: window.__sootsimFilm.b64.substr(${offset}, ${CHUNK}) }))()`, ) b64 += part.data } await evalInPage( bridge, `(() => { try { delete window.__sootsimFilm } catch {} return true })()`, ) const webmBytes = Buffer.from(b64, 'base64') mkdirSync(dirname(outPath), { recursive: true }) if (wantsMp4) { const tmpWebm = `${outPath}.tmp.webm` writeFileSync(tmpWebm, webmBytes) const ff = spawnSync( 'ffmpeg', [ '-y', '-i', tmpWebm, '-c:v', 'libx264', // libx264 + yuv420p requires even dimensions; the stage canvas is // whatever the window layout produced (e.g. 750x987) '-vf', 'scale=trunc(iw/2)*2:trunc(ih/2)*2', '-pix_fmt', 'yuv420p', '-movflags', '+faststart', outPath, ], { stdio: ['ignore', 'ignore', 'pipe'] }, ) if (ff.status !== 0) { // keep the source webm so the capture isn't lost const tail = String(ff.stderr ?? '') .split('\n') .filter(Boolean) .slice(-4) .join('\n ') console.error(` film: ffmpeg transcode failed:\n ${tail}`) console.error(` film: source kept at ${tmpWebm}`) return 1 } unlinkSync(tmpWebm) } else { writeFileSync(outPath, webmBytes) } // -- restore shell state ---------------------------------------------- if (!keepThree) { if (!before?.editor) { await evalInPage( bridge, `window.SootSim?.bridges?.settings?.set?.('threeRecordingEditor', false)`, ) } if (!before?.threeMode) { await evalInPage( bridge, `window.SootSim?.bridges?.settings?.set?.('threeMode', false)`, ) } } console.log( ` film: saved ${outPath} (${(webmBytes.length / 1e6).toFixed(1)}MB source)`, ) return 0 } finally { bridge.close() } }