/** * pi-wl-images — flicker-free inline images for Wayland + Ghostty. * * Flow: /paste (or ctrl+v) reads the Wayland clipboard, stashes the bytes, and * drops a marker in the editor. On submit, the marker is swapped for a real * image content block so the model sees the image, and a custom entry is * appended so the TUI shows it inline in the transcript. * * Rendering goes through Kitty Unicode placeholders (see placeholder.ts), which * keeps images on pi's ordinary differential-redraw path instead of the * reserved-row path that triggers full-screen repaints. */ import { readFileSync } from "node:fs"; import { basename } from "node:path"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { Text } from "@earendil-works/pi-tui"; import { readClipboardImage, extensionForMime } from "./clipboard.ts"; import { resolveCellSize, FALLBACK_CELL } from "./cell-size.ts"; import { imageDimensions, type Dimensions } from "./dimensions.ts"; import { WlImage } from "./image-component.ts"; import type { CellSize } from "./placeholder.ts"; const MARKER = "[wl-image]"; const MAX_BYTES = 10 * 1024 * 1024; interface Pending { base64: string; mimeType: string; dimensions: Dimensions; label: string; } /** Images queued by /paste but not yet sent with a message. */ const pending: Pending[] = []; let cellSize: CellSize = FALLBACK_CELL; /** * Rendered images, keyed by entry id. * * pi rebuilds a custom entry's component on every invalidate() (theme change, * expand toggle, reload). Without this cache each rebuild would mint a fresh * image id, retransmit the payload, and leave the previous copy resident in the * terminal for the rest of the session. */ const live = new Map(); function attach(bytes: Buffer, mimeType: string, label: string): Pending { if (bytes.length > MAX_BYTES) { throw new Error( `Image is ${(bytes.length / 1024 / 1024).toFixed(1)}MB, over the ${MAX_BYTES / 1024 / 1024}MB limit.`, ); } const entry: Pending = { base64: bytes.toString("base64"), mimeType, dimensions: imageDimensions(bytes), label, }; pending.push(entry); return entry; } export default function (pi: ExtensionAPI) { pi.on("session_start", async () => { // Cheap and once: Herdr panes do not answer the terminal's size query, // so this asks herdr instead. Getting it wrong mis-sizes every image. cellSize = await resolveCellSize(); }); pi.on("session_shutdown", async () => { // Free the terminal's copy of every image we transmitted. for (const image of live.values()) image.dispose(); live.clear(); }); /** Render an image inline in the chat transcript. */ pi.registerEntryRenderer("wl-image", (entry, _options, theme) => { const data = entry.data as { base64: string; label: string; dimensions: Dimensions }; if (!data?.base64) return new Text(theme.fg("dim", "[image missing]"), 1, 0); const cached = live.get(entry.id); if (cached) return cached; const image = new WlImage(data.base64, data.dimensions ?? { widthPx: 800, heightPx: 600 }, { cell: cellSize, maxWidthCells: 60, maxHeightCells: 30, fallbackText: `[image: ${data.label}]`, fallbackStyle: (text) => theme.fg("dim", text), }); live.set(entry.id, image); return image; }); async function pasteFromClipboard(ctx: ExtensionContext): Promise { let image: ReturnType; try { image = readClipboardImage(); } catch (err) { ctx.ui.notify((err as Error).message, "error"); return; } if (!image) { ctx.ui.notify("No image on the clipboard.", "warning"); return; } let entry: Pending; try { entry = attach(image.bytes, image.mimeType, `clipboard.${extensionForMime(image.mimeType)}`); } catch (err) { ctx.ui.notify((err as Error).message, "error"); return; } // The marker is what the user sees and can delete; deleting it drops the // image, which is the only sane way to cancel an attachment. const current = ctx.ui.getEditorText() ?? ""; const separator = current.length > 0 && !current.endsWith(" ") ? " " : ""; ctx.ui.setEditorText(`${current}${separator}${MARKER} `); ctx.ui.notify( `Attached ${entry.dimensions.widthPx}x${entry.dimensions.heightPx} ${entry.mimeType}`, "info", ); } pi.registerCommand("paste", { description: "Attach an image from the Wayland clipboard", handler: async (_args, ctx) => pasteFromClipboard(ctx), }); pi.registerCommand("image", { description: "Attach an image file by path", handler: async (args, ctx) => { const path = args.trim().replace(/^~(?=\/)/, process.env.HOME ?? "~"); if (!path) { ctx.ui.notify("Usage: /image ", "warning"); return; } try { const bytes = readFileSync(path); const ext = path.split(".").pop()?.toLowerCase() ?? ""; const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "webp" ? "image/webp" : ext === "gif" ? "image/gif" : "image/png"; const entry = attach(bytes, mimeType, basename(path)); const current = ctx.ui.getEditorText() ?? ""; const separator = current.length > 0 && !current.endsWith(" ") ? " " : ""; ctx.ui.setEditorText(`${current}${separator}${MARKER} `); ctx.ui.notify( `Attached ${basename(path)} (${entry.dimensions.widthPx}x${entry.dimensions.heightPx})`, "info", ); } catch (err) { ctx.ui.notify(`Could not read ${path}: ${(err as Error).message}`, "error"); } }, }); pi.registerShortcut("ctrl+v", { description: "Paste image from the Wayland clipboard", handler: async (ctx) => pasteFromClipboard(ctx), }); /** * Swap markers for real image blocks on submit. * * Markers are counted, not just stripped: if the user deleted one of them, * only that many images go out and the rest are dropped. Silently sending an * image the user removed from the draft would be worse than dropping it. */ pi.on("input", async (event) => { if (pending.length === 0) return { action: "continue" as const }; // Extension-injected input is not the user's draft, so it never carries // markers and must not consume the queue. if (event.source === "extension") return { action: "continue" as const }; const markers = event.text.split(MARKER).length - 1; if (markers === 0) return { action: "continue" as const }; // Take only what this message claims. Anything queued beyond the marker // count stays pending for the next message rather than being discarded. const attached = pending.splice(0, markers); const text = event.text.replaceAll(MARKER, "").replace(/\s+/g, " ").trim(); for (const image of attached) { // The entry itself carries the payload, so the transcript still renders // after a reload without any temp file to write, find, or clean up. pi.appendEntry("wl-image", { base64: image.base64, label: image.label, dimensions: image.dimensions, }); } return { action: "transform" as const, text: text.length > 0 ? text : "(see attached image)", // Preserve images another extension already attached. images: [ ...(event.images ?? []), ...attached.map((image) => ({ type: "image" as const, data: image.base64, mimeType: image.mimeType, })), ], }; }); }