/** * pi-loom: Context Offloader — TDAM-inspired symbolic short-term memory * * Stores large tool outputs to external files and replaces them with * lightweight symbolic references — cutting token usage while preserving * full traceability via node_id. * * Architecture: * Tool output (100K tokens) * → 1. Full text → .pi-loom/refs//.md * → 2. Symbolic summary → [REF: node_id] + brief description * → 3. Agent can grep for node_id to recover full text on demand * * Mermaid integration (optional): * Generates lightweight Mermaid task graphs from accumulated refs. */ import { existsSync, mkdirSync, readdirSync, readFileSync, rmdirSync, statSync, unlinkSync, writeFileSync, } from "node:fs"; import { join } from "node:path"; import { genId, getDbDir } from "./types.js"; /** Create a short unique node ID. */ function shortId(): string { return genId().slice(0, 8); } export interface OffloadResult { nodeId: string; filePath: string; originalBytes: number; compressedBytes: number; summary: string; refString: string; } export function offloadText(content: string, sessionId: string, label: string, maxInlineChars = 500): OffloadResult { const nodeId = shortId(); const baseDir = join(getDbDir(), "refs", sessionId); mkdirSync(baseDir, { recursive: true }); const filename = `${nodeId}-${label.replace(/[<>:"/\\|?*\s]+/g, "-").slice(0, 60)}.md`; const filePath = join(baseDir, filename); writeFileSync(filePath, content, "utf-8"); const originalBytes = Buffer.byteLength(content, "utf-8"); const inline = content.slice(0, maxInlineChars).replace(/\n/g, " ").trim(); const more = content.length > maxInlineChars ? "..." : ""; const summary = inline + more; const refString = [ `[REF:${nodeId}] ${label}: ${summary.slice(0, 120)}`, `(full: refs/${sessionId}/${filename}, ${(originalBytes / 1024).toFixed(1)}KB)`, ].join(" "); return { nodeId, filePath, originalBytes, compressedBytes: Buffer.byteLength(refString, "utf-8"), summary, refString, }; } export function retrieveOffload(nodeId: string, sessionId?: string): string | null { const refsDir = join(getDbDir(), "refs"); if (sessionId) { const sessionDir = join(refsDir, sessionId); if (existsSync(sessionDir)) { for (const f of readdirSync(sessionDir)) { if (f.startsWith(nodeId)) return readFileSync(join(sessionDir, f), "utf-8"); } } } try { for (const dir of readdirSync(refsDir)) { const sessionDir = join(refsDir, dir); try { for (const f of readdirSync(sessionDir)) { if (f.startsWith(nodeId)) return readFileSync(join(sessionDir, f), "utf-8"); } } catch { /* skip */ } } } catch { /* empty */ } return null; } export function generateMermaidGraph(sessionId: string): string { const sessionDir = join(getDbDir(), "refs", sessionId); if (!existsSync(sessionDir)) return ""; const files = readdirSync(sessionDir).filter((f) => f.endsWith(".md")); if (files.length === 0) return ""; const lines: string[] = ["```mermaid", "graph LR"]; let prevId: string | null = null; for (const f of files.sort()) { const nodeId = f.split("-")[0]; const label = f.replace(/\.md$/, "").slice(9).replace(/-/g, " ").slice(0, 30); lines.push(` ${nodeId}["${label}"]`); if (prevId) lines.push(` ${prevId} --> ${nodeId}`); prevId = nodeId; } lines.push("```"); return lines.join("\n"); } export function listSessionRefs(sessionId: string): Array<{ nodeId: string; filename: string; size: number }> { const sessionDir = join(getDbDir(), "refs", sessionId); if (!existsSync(sessionDir)) return []; return readdirSync(sessionDir) .filter((f) => f.endsWith(".md")) .map((f) => { const stat = statSync(join(sessionDir, f)); return { nodeId: f.split("-")[0], filename: f, size: stat.size }; }); } export function cleanOldRefs(retentionDays = 7): number { const refsDir = join(getDbDir(), "refs"); if (!existsSync(refsDir)) return 0; const cutoff = Date.now() - retentionDays * 24 * 3600 * 1000; let removed = 0; try { for (const dir of readdirSync(refsDir)) { const sessionDir = join(refsDir, dir); try { const files = readdirSync(sessionDir); let dirModified = false; for (const f of files) { const fileStat = statSync(join(sessionDir, f)); if (fileStat.mtimeMs <= cutoff) { unlinkSync(join(sessionDir, f)); removed++; dirModified = true; } } // Remove the session directory if all files were cleaned if (dirModified && readdirSync(sessionDir).length === 0) { rmdirSync(sessionDir); } } catch { /* skip */ } } } catch { /* empty */ } return removed; }