/** * evermind_video.ts — thin bridge that makes the EXISTING EvermindLM generate * video. No new model, no new generator: the codec turns frames into tokens, the * unqualified `EvermindLM.generate` autoregresses over them, and the codec turns * the emitted tokens back into frames. Deliberately DRY — everything the text * path already has (sampling, checkpoints, delta export, training) is reused. */ import type { EvermindLM, LMGenerateOptions } from "../lm/evermind_lm.js"; import type { VideoRVQCodec, Video, Frame } from "./video_rvq.js"; import type { ImageRVQCodec } from "./image_rvq.js"; /** * Build a unified training sequence `text… frames… ` for * `EvermindLMTrainer.fit`. `textTokens` are ids in the codec's text region (from * a BPE tokenizer whose vocab size equals `codec.vocab.textVocabSize`). Training * the LM on many such sequences is what teaches text→video. */ export function buildVideoSequence(codec: VideoRVQCodec, textTokens: number[], video: Video): number[] { return [...textTokens, ...codec.encode(video)]; } /** * Generate video from a prompt of already-tokenised context (text ids and/or a * partial video stream). The prompt should normally end just before or at the * `` marker; generation stops at `` or after `maxNewTokens`. * * Returns the decoded clip plus the raw produced token ids (useful for chaining * or continued generation). */ export function generateVideo( lm: EvermindLM, codec: VideoRVQCodec, promptTokens: number[], opts: LMGenerateOptions, ): { video: Video; tokens: number[] } { if (lm.config.vocabSize !== codec.vocabSize) { throw new Error( `generateVideo: EvermindLM vocabSize (${lm.config.vocabSize}) must equal codec.vocabSize (${codec.vocabSize})`, ); } const tokens = lm.generate(promptTokens, { ...opts, stopToken: opts.stopToken ?? codec.vocab.eosVideo }); return { video: codec.decode(tokens), tokens }; } /** Generate a single image — the still-image case of {@link generateVideo}. */ export function generateImage( lm: EvermindLM, codec: ImageRVQCodec, promptTokens: number[], opts: LMGenerateOptions, ): { image: Frame; tokens: number[] } { const { video, tokens } = generateVideo(lm, codec.video, promptTokens, opts); return { image: video[0] ?? new Float32Array(codec.frameSize), tokens }; }