/** * outpaint-keyframe — pad a keyframe to a target canvas and build an * inpainting mask for the new border region. * * Core (pad + mask) is pure/deterministic using only sharp. * Fill is optional/scaffold: if a fill() factory is provided it calls the * go-bananas REST edit endpoint; without a key it is inert. */ import sharp from 'sharp'; import { mkdir, writeFile } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import { dirname } from 'node:path'; // --------------------------------------------------------------------------- // Public types // --------------------------------------------------------------------------- export interface OutpaintKeyframeInput { /** Absolute path to the source keyframe image. */ inputPath: string; /** Absolute path to write the output PNG. */ outputPath: string; /** Target canvas width in pixels (default 1920). */ targetWidth?: number; /** Target canvas height in pixels (default 1080). */ targetHeight?: number; /** * Dilation of the mask border into the original image region, expressed as * a fraction of the smaller canvas dimension (default 0.03). * E.g. on a 1920×1080 canvas, 0.03 × 1080 ≈ 32 px of feather overlap. */ maskDilation?: number; /** * Optional fill function. When provided it receives the padded PNG buffer * and the mask PNG buffer; its returned Buffer is written to outputPath * (filled:true). When absent the padded letterbox image is written * (filled:false). */ fill?: (req: { paddedPng: Buffer; maskPng: Buffer; env?: NodeJS.ProcessEnv; fetcher?: typeof fetch; }) => Promise; env?: NodeJS.ProcessEnv; fetcher?: typeof fetch; } export interface OutpaintKeyframeResult { outputPath: string; width: number; height: number; /** true when a fill() was provided and its output was written. */ filled: boolean; } // --------------------------------------------------------------------------- // Core implementation // --------------------------------------------------------------------------- const DEFAULT_TARGET_WIDTH = 1920; const DEFAULT_TARGET_HEIGHT = 1080; const DEFAULT_MASK_DILATION_FRAC = 0.03; /** * Pad `inputPath` onto a `targetWidth`×`targetHeight` canvas (centred, * letterboxed, transparent background) and produce an L-mode mask PNG where * the original image area is black (known region) and the padded border is * white (region to fill). An optional `fill()` factory can call an * inpainting backend and receive the filled result. */ export async function outpaintKeyframe( input: OutpaintKeyframeInput, ): Promise { const { inputPath, outputPath, targetWidth = DEFAULT_TARGET_WIDTH, targetHeight = DEFAULT_TARGET_HEIGHT, maskDilation = DEFAULT_MASK_DILATION_FRAC, fill, env = process.env, fetcher = fetch, } = input; if (!existsSync(inputPath)) { throw new Error(`outpaint-keyframe: input file not found: ${inputPath}`); } // ------------------------------------------------------------------ // 1. Load source image metadata // ------------------------------------------------------------------ const src = sharp(inputPath); const meta = await src.metadata(); const srcW = meta.width ?? 0; const srcH = meta.height ?? 0; if (srcW === 0 || srcH === 0) { throw new Error(`outpaint-keyframe: could not read dimensions from ${inputPath}`); } // ------------------------------------------------------------------ // 2. Compute centred placement on the target canvas // ------------------------------------------------------------------ // Scale the source to fit inside the target, preserving aspect ratio. const scaleX = targetWidth / srcW; const scaleY = targetHeight / srcH; const scale = Math.min(scaleX, scaleY, 1); // never upscale beyond 1:1 const fittedW = Math.round(srcW * scale); const fittedH = Math.round(srcH * scale); const offsetX = Math.floor((targetWidth - fittedW) / 2); const offsetY = Math.floor((targetHeight - fittedH) / 2); // ------------------------------------------------------------------ // 3. Build the padded (letterboxed) PNG // ------------------------------------------------------------------ const resizedBuf = await sharp(inputPath) .resize(fittedW, fittedH, { fit: 'fill' }) .png() .toBuffer(); const paddedPng = await sharp({ create: { width: targetWidth, height: targetHeight, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 }, }, }) .composite([{ input: resizedBuf, left: offsetX, top: offsetY }]) .png() .toBuffer(); // ------------------------------------------------------------------ // 4. Build the RGBA alpha mask PNG // alpha 0 (transparent) = border to fill, alpha 255 (opaque) = keep. // go-bananas proxies masked edits to OpenAI gpt-image-2, which reads ONLY // the mask's alpha channel (transparent = edit, opaque = preserve), so the // mask MUST be RGBA — a grayscale luminance mask is ignored by the model. // dilation shrinks the opaque (known) region by dilationPx on each side. // ------------------------------------------------------------------ const dilationPx = Math.max( 0, Math.round(maskDilation * Math.min(targetWidth, targetHeight)), ); // Inner known rectangle (shrunk INWARD by dilationPx so the transparent fill // region bites slightly into the original image, producing a smooth seam — // standard inpainting convention). The known (opaque) rectangle is therefore // smaller than the placed image by dilationPx on each side. const innerLeft = Math.min(targetWidth, offsetX + dilationPx); const innerTop = Math.min(targetHeight, offsetY + dilationPx); const innerRight = Math.max(0, offsetX + fittedW - dilationPx); const innerBottom = Math.max(0, offsetY + fittedH - dilationPx); const innerW = innerRight - innerLeft; const innerH = innerBottom - innerTop; // Build the mask as a raw RGBA buffer: start fully transparent (fill // everywhere), then stamp the inner known rectangle opaque (alpha 255) so the // original image is preserved. RGB is irrelevant to the provider — only the // alpha channel is read — so we leave it black (0,0,0). const maskRaw = Buffer.alloc(targetWidth * targetHeight * 4, 0); // RGBA, transparent for (let row = innerTop; row < innerBottom; row++) { const rowBase = row * targetWidth; for (let col = innerLeft; col < innerRight; col++) { maskRaw[(rowBase + col) * 4 + 3] = 255; // opaque alpha = preserve } } const maskPng = await sharp(maskRaw, { raw: { width: targetWidth, height: targetHeight, channels: 4 }, }) .png() .toBuffer(); // Suppress unused variable warning: innerW / innerH are derived above and // document the composite geometry; kept for future callers. void innerW; void innerH; // ------------------------------------------------------------------ // 5. Write output (filled or letterboxed) // ------------------------------------------------------------------ await mkdir(dirname(outputPath), { recursive: true }); let filled = false; if (fill) { const filledBuf = await fill({ paddedPng, maskPng, env, fetcher }); await writeFile(outputPath, filledBuf); filled = true; } else { await writeFile(outputPath, paddedPng); } return { outputPath, width: targetWidth, height: targetHeight, filled }; } // --------------------------------------------------------------------------- // go-bananas outpaint fill factory // --------------------------------------------------------------------------- export const DEFAULT_GO_BANANAS_API_BASE = 'https://gobananasai.com/api'; /** * Masked edits go through OpenAI gpt-image-2; go-bananas rejects mask_image_id * for any non-OpenAI model, so this is the only valid default. */ export const DEFAULT_OUTPAINT_MODEL_ID = 'openai-gpt-image-2'; export const DEFAULT_OUTPAINT_PROMPT = 'Outpaint and naturally extend the scene into the transparent border ' + 'region. Seamlessly continue the existing lighting, colour, perspective, ' + 'textures and composition out to the new edges. Photorealistic, coherent ' + 'continuation with no visible seam, no new subjects, no added text, no ' + 'frame or border.'; export interface GoBananasOutpaintFillOptions { /** Outpaint instruction sent to the edit model (default DEFAULT_OUTPAINT_PROMPT). */ prompt?: string; /** * Edit model id. Must be an OpenAI model for masked edits (go-bananas refuses * mask_image_id otherwise). Default DEFAULT_OUTPAINT_MODEL_ID. */ modelId?: string; /** Optional explicit output size (e.g. '1536x1024'); omitted = provider default. */ size?: string; } interface GoBananasFillRequest { paddedPng: Buffer; maskPng: Buffer; env?: NodeJS.ProcessEnv; fetcher?: typeof fetch; } /** * Upload one PNG buffer to go-bananas' multipart edit-upload endpoint and * return its stored image id. * * POST {apiBase}/images/upload (multipart/form-data, field `file`) * → 201 { image_id, public_url, ... } */ async function uploadForEditing( png: Buffer, fileName: string, apiBase: string, apiKey: string, fetcher: typeof fetch, ): Promise { const form = new FormData(); // NOTE: do NOT set Content-Type — FormData sets the multipart boundary. form.append('file', new Blob([new Uint8Array(png)], { type: 'image/png' }), fileName); const res = await fetcher(`${apiBase}/images/upload`, { method: 'POST', headers: { 'X-API-Key': apiKey }, body: form, }); if (!res.ok) { const text = await res.text().catch(() => '(no body)'); throw new Error(`go-bananas upload (${fileName}) HTTP ${res.status}: ${text}`); } const json = (await res.json().catch(() => ({}))) as Record; const id = (json.image_id as number | undefined) ?? (json.imageId as number | undefined) ?? ((json.data as Record | undefined)?.image_id as number | undefined); if (typeof id !== 'number' || !Number.isFinite(id)) { throw new Error( `go-bananas upload (${fileName}): no image_id in response: ${JSON.stringify(json).slice(0, 200)}`, ); } return id; } /** Pull the downloadable result URL out of the edit-image response (shape-tolerant). */ function extractOutpaintResultUrl(json: Record): string | undefined { const data = json.data as Record | undefined; const candidates = [ json.fullUrl, json.full_url, json.url, json.imageUrl, json.image_url, data?.fullUrl, data?.full_url, data?.url, ]; for (const c of candidates) { if (typeof c === 'string' && c.length > 0) return c; } return undefined; } /** * Build a fill() function that performs a go-bananas masked outpaint via the * upload×2 → edit-by-id → download flow (verified against the live API): * * 1. POST {apiBase}/images/upload (padded source PNG) → image_id * 2. POST {apiBase}/images/upload (RGBA alpha mask PNG) → mask_image_id * 3. POST {apiBase}/edit-image { image_id, mask_image_id, model_id, prompt, size? } * → { fullUrl, ... } * 4. GET fullUrl → filled image bytes (returned as a Buffer) * * - apiBase from env GO_BANANAS_API_URL (default https://gobananasai.com/api) * - apiKey from env GO_BANANAS_API_KEY (throws env_var_missing when absent, * so the factory is inert in offline / CI contexts) * - injectable fetcher for offline testing */ export function goBananasOutpaintFill( options: GoBananasOutpaintFillOptions = {}, ): (req: GoBananasFillRequest) => Promise { return async ({ paddedPng, maskPng, env = process.env, fetcher = fetch }) => { const apiKey = env.GO_BANANAS_API_KEY; if (!apiKey) { throw new Error( 'env_var_missing: GO_BANANAS_API_KEY is required for go-bananas outpaint fill', ); } const apiBase = env.GO_BANANAS_API_URL?.trim() || DEFAULT_GO_BANANAS_API_BASE; const prompt = options.prompt?.trim() || DEFAULT_OUTPAINT_PROMPT; const modelId = options.modelId?.trim() || DEFAULT_OUTPAINT_MODEL_ID; // 1 + 2. Upload the padded source and the alpha mask, get their stored ids. const imageId = await uploadForEditing(paddedPng, 'padded.png', apiBase, apiKey, fetcher); const maskImageId = await uploadForEditing(maskPng, 'mask.png', apiBase, apiKey, fetcher); // 3. Edit-by-id with the mask. mask_image_id requires an OpenAI model. const editBody: Record = { image_id: imageId, mask_image_id: maskImageId, model_id: modelId, prompt, }; if (options.size) editBody.size = options.size; const editRes = await fetcher(`${apiBase}/edit-image`, { method: 'POST', headers: { 'X-API-Key': apiKey, 'Content-Type': 'application/json' }, body: JSON.stringify(editBody), }); if (!editRes.ok) { const text = await editRes.text().catch(() => '(no body)'); throw new Error(`go-bananas edit-image HTTP ${editRes.status}: ${text}`); } const editJson = (await editRes.json().catch(() => ({}))) as Record; const resultUrl = extractOutpaintResultUrl(editJson); if (!resultUrl) { throw new Error( `go-bananas edit-image: no result URL in response: ${JSON.stringify(editJson).slice(0, 300)}`, ); } // 4. Download the filled image. const dl = await fetcher(resultUrl, {}); if (!dl.ok) { throw new Error(`go-bananas outpaint download HTTP ${dl.status} for ${resultUrl}`); } return Buffer.from(await dl.arrayBuffer()); }; }