import { spawn } from "node:child_process"; import { readFileSync } from "node:fs"; import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; import { mkdir, open, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; import { homedir } from "node:os"; import { basename, dirname, extname, join } from "node:path"; import type { AddressInfo } from "node:net"; import { fileURLToPath } from "node:url"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { Box, Image, Key, Spacer, Text, deleteKittyImage, getCapabilities, getImageDimensions, matchesKey, truncateToWidth, visibleWidth, } from "@earendil-works/pi-tui"; import type { Component } from "@earendil-works/pi-tui"; interface BridgeState { available: boolean; busy: boolean; protocolVersion: number; cwd: string; sessionFile?: string; sessionId?: string; sessionName?: string; lastUserMessage?: string; lastUserMessageAt?: string; hasUI?: boolean; stdoutIsTTY?: boolean; tty?: string; heartbeatAt: string; updatedAt: string; port?: number; } interface DeliverPayload { text?: string; annotatedImageBase64: string; annotatedMediaType?: string; metadata?: unknown; } interface DeliverResult { ok: boolean; deliveryMode: "immediate" | "steer"; confirmed: boolean; confirmation: "accepted" | "queued" | "sessionUpdated"; message: string; busy: boolean; sessionFile?: string; sessionId?: string; sessionName?: string; } interface CaptureMetadata { version?: number; createdAt?: string; sourceImagePath?: string; annotatedImagePath?: string; annotations?: unknown[]; } interface CapturePreviewDetails { targetSession: string; deliveryMode: "immediate" | "steer"; sentAt: string; protocolVersion: number; annotatedMediaType: string; annotatedBytes: number; annotationCount?: number; metadataVersion?: number; metadataCreatedAt?: string; annotatedImagePath?: string; sessionFile?: string; sessionId?: string; sessionName?: string; } type CapturePreviewContentBlock = | { type: "text"; text: string } | { type: "image"; data: string; mimeType: string }; interface CaptureOption { name: string; directory: string; imagePath: string; mediaType: string; sortTimeMs: number; createdAt?: string; annotationCount?: number; metadata: Record; } interface LastUserMessagePreview { text: string; sentAt?: string; } const REGISTRY_PATH = join(homedir(), ".pi", "agent", "pilens-bridge.json"); const REGISTRIES_DIRECTORY = join(homedir(), ".pi", "agent", "pilens-bridges"); const NATIVE_REGISTRY_PATH = join(homedir(), ".pi", "agent", "pilens-native.json"); const TRIGGER_PATH = join(homedir(), ".pi", "agent", "pilens-trigger.json"); const CAPTURES_DIRECTORY = join(homedir(), "Library", "Application Support", "PiLens", "Captures"); const INSTALLED_NATIVE_APP_PATH = "/Applications/PiLens.app"; const INSTALLED_NATIVE_APP_BINARY = join(INSTALLED_NATIVE_APP_PATH, "Contents", "MacOS", "PiLens"); const PACKAGE_DIRECTORY = dirname(fileURLToPath(import.meta.url)); const NATIVE_APP_DIRECTORY = process.env.PILENS_NATIVE_APP_DIRECTORY ?? join(PACKAGE_DIRECTORY, "..", "..", "native"); const NATIVE_DEBUG_BINARY = join(NATIVE_APP_DIRECTORY, ".build", "debug", "PiLens"); const NATIVE_RELEASE_BINARY = join(NATIVE_APP_DIRECTORY, ".build", "release", "PiLens"); const HOST = "127.0.0.1"; const BRIDGE_PROTOCOL_VERSION = 2; const NATIVE_LAUNCH_POLL_MS = 500; const NATIVE_LAUNCH_GRACE_MS = 60000; const BRIDGE_HEARTBEAT_MS = 15000; const SESSION_PREVIEW_TAIL_BYTES = 256 * 1024; const SESSION_PREVIEW_MAX_LENGTH = 180; function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function parseCaptureMetadata(value: unknown): CaptureMetadata | undefined { if (!isRecord(value)) return undefined; return { version: typeof value.version === "number" ? value.version : undefined, createdAt: typeof value.createdAt === "string" ? value.createdAt : undefined, sourceImagePath: typeof value.sourceImagePath === "string" ? value.sourceImagePath : undefined, annotatedImagePath: typeof value.annotatedImagePath === "string" ? value.annotatedImagePath : undefined, annotations: Array.isArray(value.annotations) ? value.annotations : undefined, }; } function formatBytes(bytes: number): string { if (!Number.isFinite(bytes) || bytes <= 0) return "0 B"; const units = ["B", "KB", "MB", "GB"]; let value = bytes; let unitIndex = 0; while (value >= 1024 && unitIndex < units.length - 1) { value /= 1024; unitIndex += 1; } const digits = value >= 100 || unitIndex === 0 ? 0 : value >= 10 ? 1 : 2; return `${value.toFixed(digits)} ${units[unitIndex]}`; } function formatPathLine(label: string, path?: string): string { return path ? `${label}: ${path}` : `${label}: embedded image payload`; } function isSupportedImageName(name: string): boolean { return /\.(png|jpe?g|gif|webp)$/i.test(name); } function inferMediaType(imagePath: string): string { switch (extname(imagePath).toLowerCase()) { case ".jpg": case ".jpeg": return "image/jpeg"; case ".gif": return "image/gif"; case ".webp": return "image/webp"; case ".png": default: return "image/png"; } } function formatCaptureTimestamp(capture: CaptureOption): string { const date = new Date(capture.createdAt ?? capture.sortTimeMs); if (Number.isNaN(date.getTime())) return capture.name; const month = String(date.getMonth() + 1).padStart(2, "0"); const day = String(date.getDate()).padStart(2, "0"); const hour = String(date.getHours()).padStart(2, "0"); const minute = String(date.getMinutes()).padStart(2, "0"); return `${month}-${day} ${hour}:${minute}`; } function fitCell(text: string, width: number): string { const targetWidth = Math.max(0, width); const truncated = truncateToWidth(text, targetWidth, ""); return `${truncated}${" ".repeat(Math.max(0, targetWidth - visibleWidth(truncated)))}`; } async function fileExists(path: string | undefined): Promise { if (!path) return false; try { return (await stat(path)).isFile(); } catch { return false; } } async function readCaptureMetadataFile(directory: string): Promise<{ raw?: Record; parsed?: CaptureMetadata; }> { try { const raw = JSON.parse(await readFile(join(directory, "annotations.json"), "utf8")) as unknown; return { raw: isRecord(raw) ? raw : undefined, parsed: parseCaptureMetadata(raw), }; } catch { return {}; } } function extractTextContent(content: unknown): string | undefined { if (typeof content === "string") { return content; } if (!Array.isArray(content)) { return undefined; } const text = content .map((block) => (isRecord(block) && block.type === "text" && typeof block.text === "string" ? block.text : undefined)) .filter((part): part is string => Boolean(part?.trim())) .join("\n"); return text.trim() ? text : undefined; } function formatSessionPreviewText(text: string): string { const normalized = text.replace(/\s+/g, " ").trim(); if (normalized.length <= SESSION_PREVIEW_MAX_LENGTH) { return normalized; } return `${normalized.slice(0, SESSION_PREVIEW_MAX_LENGTH - 1).trimEnd()}…`; } function timestampToISOString(value: unknown): string | undefined { if (typeof value === "number" && Number.isFinite(value)) { const milliseconds = value < 1_000_000_000_000 ? value * 1000 : value; const date = new Date(milliseconds); return Number.isNaN(date.getTime()) ? undefined : date.toISOString(); } if (typeof value !== "string") { return undefined; } const trimmed = value.trim(); if (!trimmed) { return undefined; } const numericValue = Number(trimmed); if (Number.isFinite(numericValue)) { return timestampToISOString(numericValue); } const date = new Date(trimmed); return Number.isNaN(date.getTime()) ? undefined : date.toISOString(); } function extractMessageTimestamp(record: Record): string | undefined { const message = isRecord(record.message) ? record.message : undefined; return ( timestampToISOString(record.timestamp) ?? timestampToISOString(record.createdAt) ?? timestampToISOString(record.time) ?? timestampToISOString(message?.timestamp) ?? timestampToISOString(message?.createdAt) ?? timestampToISOString(message?.time) ); } async function readLastUserMessage(sessionFile?: string): Promise { if (!sessionFile) return undefined; let handle: Awaited> | undefined; try { handle = await open(sessionFile, "r"); const fileStat = await handle.stat(); const start = Math.max(0, fileStat.size - SESSION_PREVIEW_TAIL_BYTES); const byteLength = fileStat.size - start; if (byteLength <= 0) return undefined; const buffer = Buffer.alloc(byteLength); await handle.read(buffer, 0, byteLength, start); let lines = buffer.toString("utf8").split(/\r?\n/); if (start > 0) { lines = lines.slice(1); } for (let index = lines.length - 1; index >= 0; index--) { const line = lines[index]?.trim(); if (!line) continue; try { const record = JSON.parse(line) as unknown; if (!isRecord(record) || !isRecord(record.message)) continue; if (record.message.role !== "user") continue; const text = extractTextContent(record.message.content); if (text?.trim()) { return { text: formatSessionPreviewText(text), sentAt: extractMessageTimestamp(record), }; } } catch { // Ignore partial or non-JSON lines in the session log tail. } } } catch { return undefined; } finally { await handle?.close().catch(() => undefined); } return undefined; } async function findAnnotatedImagePath(directory: string, name: string, metadata?: CaptureMetadata): Promise { for (const candidate of [metadata?.annotatedImagePath, join(directory, `${name}.png`)]) { if (await fileExists(candidate)) return candidate; } try { const files = (await readdir(directory, { withFileTypes: true })) as Array<{ name: string; isFile(): boolean }>; const annotated = files.find( (file) => file.isFile() && file.name !== "original.png" && isSupportedImageName(file.name), ); if (annotated) return join(directory, annotated.name); const original = files.find((file) => file.isFile() && file.name === "original.png"); if (original) return join(directory, original.name); } catch { // Ignore unreadable capture directories. } return undefined; } function readImageBase64(path: string): string { return readFileSync(path).toString("base64"); } function expandHomePath(path: string): string { if (path === "~") return homedir(); if (path.startsWith("~/")) return join(homedir(), path.slice(2)); return path; } function isNativeMacOSScreenshotName(name: string): boolean { return /^(Screenshot|Screen Shot).+\.(png|jpe?g|gif|webp)$/i.test(name); } async function commandOutput(command: string, args: string[], timeoutMs = 1000): Promise { return new Promise((resolve) => { const child = spawn(command, args, { stdio: ["ignore", "pipe", "ignore"] }); const chunks: Buffer[] = []; let settled = false; const finish = (value: string | undefined) => { if (settled) return; settled = true; clearTimeout(timer); resolve(value); }; const timer = setTimeout(() => { child.kill(); finish(undefined); }, timeoutMs); child.stdout?.on("data", (chunk: Buffer) => chunks.push(chunk)); child.once("error", () => finish(undefined)); child.once("close", (code) => { if (code !== 0) { finish(undefined); return; } const output = Buffer.concat(chunks).toString("utf8").trim(); finish(output || undefined); }); }); } async function nativeMacOSScreenshotDirectories(): Promise { const directories = new Set([join(homedir(), "Desktop")]); const configuredLocation = await commandOutput("/usr/bin/defaults", ["read", "com.apple.screencapture", "location"]); if (configuredLocation) { directories.add(expandHomePath(configuredLocation)); } return [...directories]; } async function loadPiLensCaptures(limit?: number): Promise { let entries: Array<{ name: string; isDirectory(): boolean }>; try { entries = (await readdir(CAPTURES_DIRECTORY, { withFileTypes: true })) as Array<{ name: string; isDirectory(): boolean }>; } catch { return []; } const directories = ( await Promise.all( entries .filter((entry) => entry.isDirectory()) .map(async (entry) => { const directory = join(CAPTURES_DIRECTORY, entry.name); try { return { name: entry.name, directory, mtimeMs: (await stat(directory)).mtimeMs, }; } catch { return undefined; } }), ) ).filter((entry): entry is { name: string; directory: string; mtimeMs: number } => Boolean(entry)); const captures: CaptureOption[] = []; const maxCandidates = limit === undefined ? directories.length : Math.max(limit * 4, limit); const candidates = directories .sort((a, b) => b.mtimeMs - a.mtimeMs) .slice(0, maxCandidates); for (const candidate of candidates) { const metadataResult = await readCaptureMetadataFile(candidate.directory); const imagePath = await findAnnotatedImagePath(candidate.directory, candidate.name, metadataResult.parsed); if (!imagePath) continue; try { const imageDetails = await stat(imagePath); const createdAt = metadataResult.parsed?.createdAt; const createdAtMs = createdAt ? Date.parse(createdAt) : NaN; const sortTimeMs = Number.isFinite(createdAtMs) ? createdAtMs : imageDetails.mtimeMs; const metadata: Record = { ...(metadataResult.raw ?? {}), source: metadataResult.raw?.source ?? "pilens", annotatedImagePath: metadataResult.raw?.annotatedImagePath ?? imagePath, createdAt: metadataResult.raw?.createdAt ?? createdAt ?? new Date(imageDetails.mtimeMs).toISOString(), }; captures.push({ name: candidate.name, directory: candidate.directory, imagePath, mediaType: inferMediaType(imagePath), sortTimeMs, createdAt, annotationCount: metadataResult.parsed?.annotations?.length, metadata, }); } catch { // Ignore captures whose image disappeared while listing. } } return captures.sort((a, b) => b.sortTimeMs - a.sortTimeMs); } async function loadNativeMacOSScreenshots(limit?: number): Promise { const captures: CaptureOption[] = []; const directories = await nativeMacOSScreenshotDirectories(); const maxPerDirectory = limit === undefined ? undefined : Math.max(limit * 4, limit); for (const directory of directories) { let entries: Array<{ name: string; isFile(): boolean }>; try { entries = (await readdir(directory, { withFileTypes: true })) as Array<{ name: string; isFile(): boolean }>; } catch { continue; } const candidates = ( await Promise.all( entries .filter((entry) => entry.isFile() && isNativeMacOSScreenshotName(entry.name) && isSupportedImageName(entry.name)) .map(async (entry) => { const imagePath = join(directory, entry.name); try { return { name: entry.name, imagePath, details: await stat(imagePath) }; } catch { return undefined; } }), ) ) .filter((candidate): candidate is { name: string; imagePath: string; details: { mtimeMs: number } } => Boolean(candidate)) .sort((a, b) => b.details.mtimeMs - a.details.mtimeMs) .slice(0, maxPerDirectory); for (const candidate of candidates) { const createdAt = new Date(candidate.details.mtimeMs).toISOString(); captures.push({ name: candidate.name, directory, imagePath: candidate.imagePath, mediaType: inferMediaType(candidate.imagePath), sortTimeMs: candidate.details.mtimeMs, createdAt, metadata: { source: "macos-screenshot", createdAt, annotatedImagePath: candidate.imagePath, originalImagePath: candidate.imagePath, }, }); } } return captures.sort((a, b) => b.sortTimeMs - a.sortTimeMs); } function captureKey(capture: CaptureOption): string { return capture.imagePath; } function blendCaptureSources(piLensCaptures: CaptureOption[], nativeScreenshots: CaptureOption[], limit?: number): CaptureOption[] { if (piLensCaptures.length === 0 || nativeScreenshots.length === 0) { const sorted = [...piLensCaptures, ...nativeScreenshots].sort((a, b) => b.sortTimeMs - a.sortTimeMs); return limit === undefined ? sorted : sorted.slice(0, limit); } const visibleSourceMix = [...piLensCaptures.slice(0, 3), ...nativeScreenshots.slice(0, 2)].sort( (a, b) => b.sortTimeMs - a.sortTimeMs, ); const pinned = new Set(visibleSourceMix.map(captureKey)); const rest = [...piLensCaptures, ...nativeScreenshots] .filter((capture) => !pinned.has(captureKey(capture))) .sort((a, b) => b.sortTimeMs - a.sortTimeMs); const blended = [...visibleSourceMix, ...rest]; return limit === undefined ? blended : blended.slice(0, limit); } async function loadRecentCaptures(limit?: number): Promise { const sourceLimit = limit === undefined ? undefined : Math.max(limit * 4, limit); const [piLensCaptures, nativeScreenshots] = await Promise.all([ loadPiLensCaptures(sourceLimit), loadNativeMacOSScreenshots(sourceLimit), ]); return blendCaptureSources(piLensCaptures, nativeScreenshots, limit); } class CapturePickerComponent implements Component { private selectedIndex = 0; private scrollOffset = 0; private cachedImageKey?: string; private cachedImageLines?: string[]; private readonly base64Cache = new Map(); private readonly listVisibleItems = 5; private readonly previewImageId = 9100; private lastRenderedFrameKey = ""; constructor( private readonly captures: CaptureOption[], private readonly theme: any, private readonly done: (capture: CaptureOption | null) => void, ) {} handleInput(data: string): void { if (matchesKey(data, Key.up) || matchesKey(data, Key.left) || data === "k" || data === "h") { this.moveSelection(-1); return; } if (matchesKey(data, Key.down) || matchesKey(data, Key.right) || data === "j" || data === "l") { this.moveSelection(1); return; } if (matchesKey(data, Key.enter)) { this.finish(this.captures[this.selectedIndex] ?? null); return; } if (matchesKey(data, Key.escape) || matchesKey(data, Key.ctrl("c"))) { this.finish(null); return; } if (/^[1-5]$/.test(data)) { const index = this.scrollOffset + Number.parseInt(data, 10) - 1; const capture = this.captures[index]; if (capture) { this.selectedIndex = index; this.finish(capture); } } } render(width: number): string[] { const safeWidth = Math.max(1, width); const listWidth = Math.min(44, Math.max(26, Math.floor(safeWidth * 0.38))); const previewWidth = Math.max(1, safeWidth - listWidth - 3); const previewLines = 14; const selected = this.captures[this.selectedIndex]; const imageLines = selected ? this.renderSelectedImageLines(selected, previewWidth, previewLines) : []; const contentRows = previewLines; const countInfo = this.captures.length > 0 ? ` (${this.selectedIndex + 1}/${this.captures.length})` : ""; const lines = [ truncateToWidth(this.theme.fg("accent", this.theme.bold(`Recent captures${countInfo}`)), safeWidth), truncateToWidth( this.theme.fg("dim", "↑/↓ move • 1-5 send visible row • enter send selected • esc cancel"), safeWidth, ), ]; if (!getCapabilities().images) { lines.push( truncateToWidth( this.theme.fg("warning", "Inline image preview is unavailable in this terminal; showing the selected capture path."), safeWidth, ), ); } lines.push(""); for (let row = 0; row < contentRows; row++) { const visibleIndex = this.scrollOffset + row; const capture = row < this.listVisibleItems ? this.captures[visibleIndex] : undefined; const listLine = capture ? this.renderListCell(capture, visibleIndex, listWidth) : fitCell("", listWidth); const rawImageLine = imageLines[row] ?? ""; const imageLine = this.truncateNonImageLine(rawImageLine, previewWidth); lines.push(`${listLine}${this.theme.fg("muted", "│")} ${imageLine}`); } lines.push(""); lines.push( truncateToWidth( this.theme.fg("success", "Enter sends the selected capture to this pi session now."), safeWidth, ), ); lines.push( truncateToWidth( this.theme.fg("dim", `selected: ${selected?.imagePath ?? "none"}`), safeWidth, ), ); return lines; } invalidate(): void { this.cachedImageKey = undefined; this.cachedImageLines = undefined; } private moveSelection(delta: number): void { if (this.captures.length === 0) return; this.selectedIndex = Math.max(0, Math.min(this.captures.length - 1, this.selectedIndex + delta)); if (this.selectedIndex < this.scrollOffset) { this.scrollOffset = this.selectedIndex; } if (this.selectedIndex >= this.scrollOffset + this.listVisibleItems) { this.scrollOffset = this.selectedIndex - this.listVisibleItems + 1; } this.invalidate(); } private renderListCell(capture: CaptureOption, index: number, width: number): string { const cursor = index === this.selectedIndex ? "▸" : " "; const source = capture.metadata.source === "macos-screenshot" ? "macOS" : "PiLens"; const summary = `${cursor} ${index - this.scrollOffset + 1} ${source} ${formatCaptureTimestamp(capture)}`; const padded = fitCell(summary, width); if (index === this.selectedIndex) { return this.theme.bg("selectedBg", this.theme.fg("accent", padded)); } return this.theme.fg("muted", padded); } private renderSelectedImageLines(capture: CaptureOption, width: number, previewLines: number): string[] { const key = `${capture.imagePath}:${width}:${previewLines}:${getCapabilities().images ?? "none"}`; if (this.cachedImageKey === key && this.cachedImageLines) return this.cachedImageLines; const deletePrevious = this.lastRenderedFrameKey && this.lastRenderedFrameKey !== key && getCapabilities().images === "kitty" ? deleteKittyImage(this.previewImageId) : ""; this.lastRenderedFrameKey = key; const base64 = this.getCaptureBase64(capture); if (!base64) { const lines = [deletePrevious + this.theme.fg("error", `Unable to read ${capture.imagePath}`)]; while (lines.length < previewLines) lines.push(""); this.cachedImageLines = lines; this.cachedImageKey = key; return lines; } const maxWidthCells = Math.max(1, Math.min(80, width)); const dimensions = getImageDimensions(base64, capture.mediaType); const constrainedWidth = dimensions ? this.calculateConstrainedImageWidth(dimensions.widthPx, dimensions.heightPx, previewLines, maxWidthCells) : maxWidthCells; const image = new Image( base64, capture.mediaType, { fallbackColor: (str: string) => this.theme.fg("muted", str) }, { maxWidthCells: constrainedWidth, filename: capture.imagePath, imageId: this.previewImageId }, ); const rendered = image.render(constrainedWidth + 2); const lines = Array.from({ length: previewLines }, (_, index) => rendered[index] ?? ""); if (lines.length > 0) lines[0] = deletePrevious + lines[0]; this.cachedImageLines = lines; this.cachedImageKey = key; return lines; } private getCaptureBase64(capture: CaptureOption): string | undefined { const cached = this.base64Cache.get(capture.imagePath); if (cached) return cached; try { const base64 = readImageBase64(capture.imagePath); this.base64Cache.set(capture.imagePath, base64); return base64; } catch { return undefined; } } private calculateConstrainedImageWidth(widthPx: number, heightPx: number, maxRows: number, maxWidthCells: number): number { const cellWidthPx = 9; const cellHeightPx = 18; const scaledWidthPx = maxWidthCells * cellWidthPx; const scale = scaledWidthPx / widthPx; const rows = Math.ceil((heightPx * scale) / cellHeightPx); if (rows <= maxRows) return maxWidthCells; const targetHeightPx = maxRows * cellHeightPx; const targetScale = targetHeightPx / heightPx; return Math.max(1, Math.min(maxWidthCells, Math.floor((widthPx * targetScale) / cellWidthPx))); } private truncateNonImageLine(line: string, width: number): string { if (line.includes("\x1b_G") || line.includes("\x1b]1337;File=")) return line; return truncateToWidth(line, width, ""); } private finish(capture: CaptureOption | null): void { this.cleanupImages(); this.done(capture); } private cleanupImages(): void { if (getCapabilities().images !== "kitty") return; try { process.stdout.write(deleteKittyImage(this.previewImageId)); } catch { // Best-effort terminal image cleanup. } } } function isPidAlive(pid: number): boolean { try { process.kill(pid, 0); return true; } catch { return false; } } async function readNativeRegistry(): Promise<{ pid?: number; triggerPath?: string } | undefined> { try { return JSON.parse(await readFile(NATIVE_REGISTRY_PATH, "utf8")) as { pid?: number; triggerPath?: string }; } catch { return undefined; } } async function pathMtimeMs(path: string): Promise { try { return (await stat(path)).mtimeMs; } catch { return 0; } } async function treeMtimeMs(path: string): Promise { try { const details = await stat(path); if (!details.isDirectory()) { return details.mtimeMs; } let latest = details.mtimeMs; for (const entry of await readdir(path, { withFileTypes: true })) { if (entry.name === ".build") continue; latest = Math.max(latest, await treeMtimeMs(join(path, entry.name))); } return latest; } catch { return 0; } } async function hasNativeSourceDirectory(): Promise { return Boolean(await pathMtimeMs(join(NATIVE_APP_DIRECTORY, "Package.swift"))); } async function latestNativeSourceMtimeMs(): Promise { if (!(await hasNativeSourceDirectory())) return 0; return Math.max( await pathMtimeMs(join(NATIVE_APP_DIRECTORY, "Package.swift")), await pathMtimeMs(join(NATIVE_APP_DIRECTORY, "Package.resolved")), await treeMtimeMs(join(NATIVE_APP_DIRECTORY, "Sources")), ); } async function spawnDetached(command: string, args: string[], cwd: string): Promise { await new Promise((resolve, reject) => { const child = spawn(command, args, { cwd, detached: true, stdio: "ignore", }); child.once("error", reject); child.once("spawn", () => { child.unref(); resolve(); }); }); } function json(response: ServerResponse, statusCode: number, payload: unknown): void { response.writeHead(statusCode, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store", }); response.end(`${JSON.stringify(payload)}\n`); } async function readJsonBody(request: IncomingMessage): Promise { const chunks: Buffer[] = []; for await (const chunk of request) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); } return JSON.parse(Buffer.concat(chunks).toString("utf8")) as T; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } export default function piCaptureBridge(pi: ExtensionAPI) { let server: Server | undefined; let ownerToken: string | undefined; let registryFilePath: string | undefined; let nativeLaunchInProgress = false; let heartbeatTimer: ReturnType | undefined; const initialHeartbeatAt = new Date().toISOString(); let state: BridgeState = { available: false, busy: false, protocolVersion: BRIDGE_PROTOCOL_VERSION, cwd: process.cwd(), heartbeatAt: initialHeartbeatAt, updatedAt: initialHeartbeatAt, }; const isNativeAppRunning = async (): Promise => { const nativeRegistry = await readNativeRegistry(); return Boolean(nativeRegistry?.pid && isPidAlive(nativeRegistry.pid)); }; const watchNativeLaunch = () => { const deadline = Date.now() + NATIVE_LAUNCH_GRACE_MS; void (async () => { while (Date.now() < deadline) { if (await isNativeAppRunning()) { nativeLaunchInProgress = false; return; } await sleep(NATIVE_LAUNCH_POLL_MS); } nativeLaunchInProgress = false; })(); }; const launchNativeAppIfNeeded = async (): Promise<{ launched: boolean; launchMethod?: string; running: boolean; launching: boolean; }> => { if (await isNativeAppRunning()) { nativeLaunchInProgress = false; return { launched: false, running: true, launching: false }; } if (nativeLaunchInProgress) { return { launched: false, running: false, launching: true }; } const nativeSourceAvailable = await hasNativeSourceDirectory(); const latestSourceMtime = nativeSourceAvailable ? await latestNativeSourceMtimeMs() : 0; const installedBinaryMtime = await pathMtimeMs(INSTALLED_NATIVE_APP_BINARY); const debugBinaryMtime = nativeSourceAvailable ? await pathMtimeMs(NATIVE_DEBUG_BINARY) : 0; const releaseBinaryMtime = nativeSourceAvailable ? await pathMtimeMs(NATIVE_RELEASE_BINARY) : 0; const freshBinary = debugBinaryMtime && debugBinaryMtime >= latestSourceMtime ? NATIVE_DEBUG_BINARY : releaseBinaryMtime && releaseBinaryMtime >= latestSourceMtime ? NATIVE_RELEASE_BINARY : undefined; const fallbackBinary = debugBinaryMtime ? NATIVE_DEBUG_BINARY : releaseBinaryMtime ? NATIVE_RELEASE_BINARY : undefined; const launchCandidates: Array<{ launchMethod: string; start: () => Promise }> = []; const launchCwd = nativeSourceAvailable ? NATIVE_APP_DIRECTORY : homedir(); if (installedBinaryMtime && installedBinaryMtime >= latestSourceMtime) { launchCandidates.push({ launchMethod: "installed app (/Applications/PiLens.app)", start: () => spawnDetached("open", [INSTALLED_NATIVE_APP_PATH], launchCwd), }); } if (nativeSourceAvailable) { if (freshBinary) { launchCandidates.push({ launchMethod: `native binary (${basename(freshBinary)})`, start: () => spawnDetached(freshBinary, [], NATIVE_APP_DIRECTORY), }); } launchCandidates.push({ launchMethod: "swift run PiLens", start: () => spawnDetached("swift", ["run", "PiLens"], NATIVE_APP_DIRECTORY), }); if (fallbackBinary && fallbackBinary !== freshBinary) { launchCandidates.push({ launchMethod: `existing binary (${basename(fallbackBinary)})`, start: () => spawnDetached(fallbackBinary, [], NATIVE_APP_DIRECTORY), }); } } if (launchCandidates.length === 0) { throw new Error("PiLens Mac app is not installed. Download it from https://pilens.dev/download and open it once, then run /pilens again."); } nativeLaunchInProgress = true; let lastError: unknown; for (const candidate of launchCandidates) { try { await candidate.start(); watchNativeLaunch(); return { launched: true, launchMethod: candidate.launchMethod, running: false, launching: false, }; } catch (error) { lastError = error; } } nativeLaunchInProgress = false; throw lastError instanceof Error ? lastError : new Error("Failed to launch the native PiLens app."); }; const writeRegistry = async () => { if (!state.port || !ownerToken) return; const payload = `${JSON.stringify({ ownerToken, host: HOST, pid: process.pid, ...state }, null, "\t")}\n`; await mkdir(join(homedir(), ".pi", "agent"), { recursive: true }); await mkdir(REGISTRIES_DIRECTORY, { recursive: true }); registryFilePath = registryFilePath ?? join(REGISTRIES_DIRECTORY, `${ownerToken}.json`); await writeFile(registryFilePath, payload, "utf8"); await writeFile(REGISTRY_PATH, payload, "utf8"); }; const removeRegistryIfOwned = async () => { if (registryFilePath) { await rm(registryFilePath, { force: true }); registryFilePath = undefined; } if (!ownerToken) return; try { const existing = JSON.parse(await readFile(REGISTRY_PATH, "utf8")) as { ownerToken?: string }; if (existing.ownerToken === ownerToken) { await rm(REGISTRY_PATH, { force: true }); } } catch { // Ignore missing or invalid registry files. } }; const refreshState = async (ctx?: ExtensionContext) => { const now = new Date().toISOString(); const sessionFile = ctx?.sessionManager.getSessionFile() ?? state.sessionFile; const lastUserMessage = await readLastUserMessage(sessionFile); state = { ...state, available: true, busy: ctx ? !ctx.isIdle() : state.busy, protocolVersion: BRIDGE_PROTOCOL_VERSION, cwd: ctx?.cwd ?? state.cwd, sessionFile, sessionId: (ctx?.sessionManager as any)?.getSessionId?.() ?? state.sessionId, sessionName: (ctx?.sessionManager as any)?.getSessionName?.() ?? state.sessionName, lastUserMessage: lastUserMessage?.text, lastUserMessageAt: lastUserMessage?.sentAt, hasUI: ctx?.hasUI ?? state.hasUI, stdoutIsTTY: Boolean(process.stdout.isTTY), tty: process.env.TTY, heartbeatAt: now, updatedAt: now, }; await writeRegistry(); }; const startHeartbeat = () => { if (heartbeatTimer) return; heartbeatTimer = setInterval(() => { void refreshState(); }, BRIDGE_HEARTBEAT_MS); heartbeatTimer.unref?.(); }; const stopHeartbeat = () => { if (!heartbeatTimer) return; clearInterval(heartbeatTimer); heartbeatTimer = undefined; }; const describeTargetSession = () => { if (state.sessionName?.trim()) { return state.sessionName.trim(); } const projectName = basename(state.cwd) || state.cwd; if (state.sessionId) { return `${projectName} • ${state.sessionId.slice(0, 8)}`; } if (state.sessionFile) { return `${projectName} • ${basename(state.sessionFile)}`; } return projectName; }; const buildResponseBase = () => ({ ok: true, busy: state.busy, protocolVersion: BRIDGE_PROTOCOL_VERSION, sessionFile: state.sessionFile, sessionId: state.sessionId, sessionName: state.sessionName, }) as const; const buildCapturePreviewContent = ( payload: DeliverPayload, promptText: string | undefined, annotatedMimeType: string, ): CapturePreviewContentBlock[] => { const blocks: CapturePreviewContentBlock[] = []; const trimmedPrompt = promptText?.trim(); if (trimmedPrompt) { blocks.push({ type: "text", text: trimmedPrompt }); } blocks.push({ type: "image", data: payload.annotatedImageBase64, mimeType: annotatedMimeType }); return blocks; }; pi.registerMessageRenderer("pilens-preview", (message, { expanded }, theme) => { const details = message.details as CapturePreviewDetails | undefined; const contentBlocks: CapturePreviewContentBlock[] = typeof message.content === "string" ? [{ type: "text", text: message.content }] : (message.content as CapturePreviewContentBlock[]); const contentImageBlocks = contentBlocks.filter((block): block is Extract => block.type === "image"); let filePreviewImageBlock: Extract | undefined; if (contentImageBlocks.length === 0 && details?.annotatedImagePath) { try { filePreviewImageBlock = { type: "image", data: readFileSync(details.annotatedImagePath).toString("base64"), mimeType: details.annotatedMediaType, }; } catch { filePreviewImageBlock = undefined; } } const imageBlocks = filePreviewImageBlock ? [...contentImageBlocks, filePreviewImageBlock] : contentImageBlocks; const textBlocks = contentBlocks.filter((block): block is Extract => block.type === "text"); const deliveryMode = details?.deliveryMode ?? "immediate"; const color = deliveryMode === "steer" ? "warning" : "success"; const heading = deliveryMode === "steer" ? "QUEUED" : "SENT"; const lines = [ `${theme.fg(color, theme.bold(`[pilens ${heading}]`))} ${details?.targetSession ?? "Pi session"}`, theme.fg( "muted", deliveryMode === "steer" ? "Screenshot queued as a user message." : imageBlocks.length > 0 ? "Screenshot delivered as the final annotated image." : "Screenshot delivered as a custom message.", ), ]; if (details) { lines.push( `${theme.fg("accent", "annotated")} • ${formatBytes(details.annotatedBytes)} • ${details.annotatedMediaType}`, formatPathLine(" path", details.annotatedImagePath), ); const metadataBits = [ details.annotationCount !== undefined ? `${details.annotationCount} annotation${details.annotationCount === 1 ? "" : "s"}` : undefined, details.metadataVersion !== undefined ? `metadata v${details.metadataVersion}` : undefined, `bridge v${details.protocolVersion}`, ] .filter(Boolean) .join(" • "); lines.push(theme.fg("dim", metadataBits)); if (expanded) { lines.push("", theme.fg("dim", JSON.stringify(details, null, 2))); } } const box = new Box(1, 1, (text) => theme.bg("customMessageBg", text)); box.addChild(new Text(lines.join("\n"), 0, 0)); if (textBlocks.length > 0 && imageBlocks.length === 0) { box.addChild(new Spacer(1)); box.addChild(new Text(textBlocks.map((block) => block.text).join("\n"), 0, 0)); } if (imageBlocks.length > 0) { box.addChild(new Spacer(1)); imageBlocks.forEach((block, index) => { const imagePath = index === 0 ? details?.annotatedImagePath : undefined; box.addChild( new Text( theme.fg("accent", imagePath ? `annotated • ${imagePath}` : "annotated"), 0, 0, ), ); box.addChild( new Image( block.data, block.mimeType, { fallbackColor: (str) => theme.fg("muted", str) }, { filename: imagePath }, ), ); if (index < imageBlocks.length - 1) { box.addChild(new Spacer(1)); } }); } return box; }); const emitCapturePreview = (payload: DeliverPayload, details: CapturePreviewDetails) => { pi.sendMessage( { customType: "pilens-preview", content: "PiLens screenshot preview — image also sent as a user message.", display: true, details, }, details.deliveryMode === "steer" ? { deliverAs: "steer" } : undefined, ); }; const sendCaptureUserMessage = (payload: DeliverPayload, details: CapturePreviewDetails) => { const content = buildCapturePreviewContent( payload, payload.text, details.annotatedMediaType, ); emitCapturePreview(payload, details); if (details.deliveryMode === "steer") { pi.sendUserMessage(content, { deliverAs: "steer" }); return; } pi.sendUserMessage(content); }; const buildCapturePreviewDetails = ( payload: DeliverPayload, targetSession: string, deliveryMode: "immediate" | "steer", ): CapturePreviewDetails => { const metadata = parseCaptureMetadata(payload.metadata); return { targetSession, deliveryMode, sentAt: new Date().toISOString(), protocolVersion: BRIDGE_PROTOCOL_VERSION, annotatedMediaType: payload.annotatedMediaType ?? "image/png", annotatedBytes: Buffer.byteLength(payload.annotatedImageBase64, "base64"), annotationCount: metadata?.annotations?.length, metadataVersion: metadata?.version, metadataCreatedAt: metadata?.createdAt, annotatedImagePath: metadata?.annotatedImagePath, sessionFile: state.sessionFile, sessionId: state.sessionId, sessionName: state.sessionName, }; }; const deliverToSession = async (payload: DeliverPayload): Promise => { const targetSession = describeTargetSession(); const responseBase = buildResponseBase(); if (state.busy) { const details = buildCapturePreviewDetails(payload, targetSession, "steer"); sendCaptureUserMessage(payload, details); return { ...responseBase, deliveryMode: "steer", confirmed: true, confirmation: "queued", message: `Queued for ${targetSession}. Pi is busy right now, so the screenshot will be delivered as a user message at the next interruption point.`, }; } const details = buildCapturePreviewDetails(payload, targetSession, "immediate"); sendCaptureUserMessage(payload, details); return { ...responseBase, deliveryMode: "immediate", confirmed: true, confirmation: "accepted", message: state.sessionFile ? `Sent to pi for ${targetSession} as a user message. The screenshot should appear in this session shortly.` : `Sent to pi for ${targetSession} as a user message. This session is not persisted, so file confirmation is unavailable.`, }; }; const ensureServer = async (ctx: ExtensionContext) => { if (server) { await refreshState(ctx); startHeartbeat(); return; } ownerToken = `${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2, 10)}`; server = createServer(async (request, response) => { try { if (!request.url) { json(response, 404, { error: "Missing URL." }); return; } if (request.method === "GET" && request.url === "/status") { await refreshState(ctx); json(response, 200, state); return; } if (request.method === "POST" && request.url === "/deliver") { await refreshState(ctx); if (!state.available) { json(response, 503, { ok: false, error: "No active pi bridge is available. Open pi and /reload the pilens extension first." }); return; } const payload = await readJsonBody(request); if (!payload.annotatedImageBase64) { json(response, 400, { ok: false, error: "The annotated image payload is required." }); return; } const delivery = await deliverToSession(payload); json(response, 200, delivery); return; } json(response, 404, { error: "Not found." }); } catch (error) { json(response, 500, { error: error instanceof Error ? error.message : String(error), }); } }); await new Promise((resolve, reject) => { server!.once("error", reject); server!.listen(0, HOST, () => { server!.off("error", reject); resolve(); }); }); const address = server.address() as AddressInfo | null; if (!address?.port) { throw new Error("Failed to determine pilens bridge port."); } state.port = address.port; await refreshState(ctx); startHeartbeat(); }; const triggerNativeCapture = async () => { await mkdir(join(homedir(), ".pi", "agent"), { recursive: true }); const requestId = `${Date.now()}-${Math.random().toString(16).slice(2, 10)}`; await writeFile( TRIGGER_PATH, `${JSON.stringify({ requestId, action: "capture", createdAt: new Date().toISOString() }, null, "\t")}\n`, "utf8", ); return requestId; }; const showStatus = async (ctx: ExtensionContext) => { await refreshState(ctx); const nativeRunning = await isNativeAppRunning(); ctx.ui.notify( state.port ? `Pi bridge ready on ${HOST}:${state.port}${nativeRunning ? " • native app detected" : nativeLaunchInProgress ? " • starting native app" : " • native app not detected"} • target ${describeTargetSession()}${state.busy ? " • busy (captures queue)" : " • idle"}` : "PiLens bridge is not running.", state.port ? "success" : "warning", ); }; const triggerLens = async (ctx: ExtensionContext) => { await ensureServer(ctx); let launchError: unknown; try { await launchNativeAppIfNeeded(); } catch (error) { launchError = error; } await triggerNativeCapture(); // Avoid emitting a success/warning toast here: the native app may not start the // capture overlay until the next trigger-file poll, and the toast can leak into // the screenshot the user is trying to take. if (launchError) { ctx.ui.notify( `Queued a capture request at ${TRIGGER_PATH}, but PiLens could not auto-start: ${launchError instanceof Error ? launchError.message : String(launchError)}`, "warning", ); } }; const showCapturePicker = async (ctx: ExtensionContext) => { if (!ctx.hasUI) { ctx.ui.notify("/pilens requires the interactive TUI to show recent captures.", "warning"); return; } await ensureServer(ctx); const captures = await loadRecentCaptures(); if (captures.length === 0) { ctx.ui.notify(`No PiLens captures or native macOS screenshots found.`, "warning"); return; } const selected = await ctx.ui.custom((tui, theme, _keybindings, done) => { const picker = new CapturePickerComponent(captures, theme, done); return { render: (width: number) => picker.render(width), invalidate: () => picker.invalidate(), handleInput: (data: string) => { picker.handleInput(data); tui.requestRender(); }, }; }); if (!selected) { ctx.ui.notify("PiLens capture selection cancelled.", "info"); return; } await refreshState(ctx); let selectedBase64: string; try { selectedBase64 = readImageBase64(selected.imagePath); } catch (error) { ctx.ui.notify(`Unable to read selected PiLens capture: ${error instanceof Error ? error.message : String(error)}`, "error"); return; } const delivery = await deliverToSession({ annotatedImageBase64: selectedBase64, annotatedMediaType: selected.mediaType, metadata: selected.metadata, }); ctx.ui.notify(delivery.message, delivery.deliveryMode === "steer" ? "warning" : "success"); }; pi.on("session_start", async (_event, ctx) => { if (!ctx.hasUI) return; await ensureServer(ctx); }); pi.on("agent_start", async (_event, ctx) => { state.busy = true; await refreshState(ctx); }); pi.on("agent_end", async (_event, ctx) => { state.busy = false; await refreshState(ctx); }); pi.on("session_switch", async (_event, ctx) => { await refreshState(ctx); }); pi.on("model_select", async (_event, ctx) => { await refreshState(ctx); }); pi.on("session_shutdown", async () => { stopHeartbeat(); state.available = false; state.updatedAt = new Date().toISOString(); state.heartbeatAt = state.updatedAt; await removeRegistryIfOwned(); if (server) { await new Promise((resolve) => server!.close(() => resolve())); server = undefined; } }); pi.registerCommand("pilens", { description: "Show recent PiLens captures and send the selected capture", handler: async (args, ctx) => { const trimmed = args.trim(); if (!trimmed) { await showCapturePicker(ctx); return; } const [subcommand] = trimmed.split(/\s+/); switch (subcommand) { case "capture": await triggerLens(ctx); return; case "captures": case "view": case "recent": case "list": await showCapturePicker(ctx); return; case "status": await showStatus(ctx); return; default: ctx.ui.notify("Usage: /pilens [capture|captures|status]", "warning"); return; } }, }); }